1 /*- 2 Copyright (C) 2008, 2010, 2011, 2013, 2014, 2022 Lucas Holt All rights reserved. 3 4 Redistribution and use in source and binary forms, with or without 5 modification, are permitted provided that the following conditions 6 are met: 7 1. Redistributions of source code must retain the above copyright 8 notice, this list of conditions and the following disclaimer. 9 2. Redistributions in binary form must reproduce the above copyright 10 notice, this list of conditions and the following disclaimer in the 11 documentation and/or other materials provided with the distribution. 12 13 THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND 14 ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16 ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE 17 FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18 DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19 OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20 HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21 LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22 OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23 SUCH DAMAGE. 24 */ 25 26 #include <iostream> 27 #include <pqxx/pqxx> 28 #include <cstring> 29 #include <string> 30 31 using namespace std; 32 using namespace pqxx; 33 34 #include <stdio.h> 35 #include <stdlib.h> 36 #include <unistd.h> 37 #include <err.h> 38 39 #include <sys/types.h> 40 #include <sha256.h> 41 42 #include <ucl.h> 43 #ifdef ucl_object_find_key 44 #undef ucl_object_find_key 45 #endif 46 extern "C" const ucl_object_t *ucl_object_find_key(const ucl_object_t *, const char *); 47 48 #include "sqlite3.h" 49 50 const char *CONFIG_FILE = "/usr/magus/config.yaml"; 51 52 struct MagusConfig { 53 string db_host; 54 string db_name; 55 string db_user; 56 string db_pass; 57 }; 58 59 MagusConfig load_config(const char *); 60 string config_string(const ucl_object_t *, const char *, const char *); 61 62 /* SQLITE3 */ 63 sqlite3* open_indexdb(int); 64 void close_indexdb(sqlite3 *); 65 int exec_indexdb(sqlite3 *, const char *, ...); 66 void create_indexdb(sqlite3 *); 67 void load_depends(sqlite3_stmt *, connection &, int, const char *, const char *); 68 69 int 70 main(int argc, char *argv[]) 71 { 72 char query_def[2048]; 73 int runid; 74 sqlite3 *db = NULL; 75 sqlite3_stmt *stmt; 76 sqlite3_stmt *depends_stmt = NULL; 77 char *fileHash; 78 char *filePath; 79 80 if (argc != 3) 81 { 82 fprintf( stderr, "Usage: %s <run id> <files>\n", argv[0] ); 83 exit(1); 84 } 85 86 runid = atoi(argv[1]); 87 if (runid < 1) 88 { 89 fprintf( stderr, "Invalid run id %d\n", runid); 90 exit(1); 91 } 92 93 MagusConfig config = load_config(CONFIG_FILE); 94 95 try { 96 97 string connect_string = "dbname=" + config.db_name + 98 " user=" + config.db_user + 99 " host=" + config.db_host + 100 " port=5432"; 101 102 if (!config.db_pass.empty()) 103 connect_string += " password=" + config.db_pass; 104 105 connection C(connect_string); 106 connection C2(connect_string); 107 108 if (C.is_open()) { 109 cout << "We are connected to " << C.dbname() << endl; 110 } else { 111 cout << "We are not connected! Check username and password." << endl; 112 return -1; 113 } 114 115 if (!C2.is_open()) { 116 cout << "Secondary connection failed! Check username and password." << endl; 117 return -1; 118 } 119 120 snprintf(query_def, sizeof(query_def), 121 "select pkgname, name, license, description, CONCAT(CONCAT_WS( '-', pkgname, version),'.mport'), version, restricted from ports where run=%d AND status!='internal' AND status!='untested' AND status!='fail' ORDER BY pkgname;", runid); 122 123 nontransaction N(C); 124 125 result R(N.exec(string(query_def))); 126 127 if (!R.empty()) 128 { 129 130 cout << "Init index db file" << endl; 131 db = open_indexdb(runid); 132 create_indexdb(db); 133 exec_indexdb(db, "BEGIN TRANSACTION"); 134 135 if (sqlite3_prepare_v2(db, 136 "INSERT INTO depends (pkg, version, d_pkg, d_version) VALUES(?,?,?,?)", 137 -1, &depends_stmt, 0) != SQLITE_OK) 138 { 139 errx(1, "Could not prepare statement"); 140 } 141 142 for (result::const_iterator row = R.begin(); row != R.end(); ++row) 143 { 144 filePath = NULL; 145 fileHash = NULL; 146 try { 147 148 string ln = row[0].as(string()) + ": " + row[1].as(string()) + " " + row[2].as(string()) + " " + row[3].as(string()) + " " + row[5].as(string()) + " " + row[4].as(string()); 149 if (asprintf(&filePath, "%s/%s", argv[2], row[4].as(string()).c_str()) == -1) 150 errx(1, "Could not allocate file path"); 151 fileHash = SHA256_File(filePath, NULL); 152 if (fileHash == NULL) 153 { 154 fprintf(stderr, "Could not locate file %s\n", filePath); 155 free(filePath); 156 filePath = NULL; 157 continue; 158 } 159 160 if (row[6].as(bool())) 161 { 162 fprintf(stderr, "File %s is restricted and will be removed.\n", filePath); 163 unlink(filePath); 164 free(fileHash); 165 free(filePath); 166 fileHash = NULL; 167 filePath = NULL; 168 continue; 169 } 170 171 { 172 if (sqlite3_prepare_v2(db, 173 "INSERT INTO packages (pkg, version, license, comment, bundlefile, hash, type) VALUES(?,?,?,?,?,?,?)", 174 -1, &stmt, 0) != SQLITE_OK) 175 { 176 errx(1, "Could not prepare statement"); 177 } 178 sqlite3_bind_text(stmt, 1, row[0].as(string()).c_str(), row[0].as(string()).length(), SQLITE_TRANSIENT); 179 sqlite3_bind_text(stmt, 2, row[5].as(string()).c_str(), row[5].as(string()).length(), SQLITE_TRANSIENT); 180 sqlite3_bind_text(stmt, 3, row[2].as(string()).c_str(), row[2].as(string()).length(), SQLITE_TRANSIENT); 181 sqlite3_bind_text(stmt, 4, row[3].as(string()).c_str(), row[3].as(string()).length(), SQLITE_TRANSIENT); 182 sqlite3_bind_text(stmt, 5, row[4].as(string()).c_str(), row[4].as(string()).length(), SQLITE_TRANSIENT); 183 sqlite3_bind_text(stmt, 6, fileHash, strlen(fileHash), SQLITE_TRANSIENT); 184 sqlite3_bind_int(stmt, 7, 0); // TODO: handle system packages? 185 186 if (sqlite3_step(stmt) != SQLITE_DONE) 187 errx(1,"Could not execute query"); 188 sqlite3_reset(stmt); 189 sqlite3_finalize(stmt); 190 free(filePath); 191 free(fileHash); 192 filePath = NULL; 193 fileHash = NULL; 194 195 if (sqlite3_prepare_v2(db, "INSERT INTO aliases (alias, pkg) VALUES(?,?)", -1, &stmt, 0) != SQLITE_OK) 196 { 197 errx(1, "Could not prepare statement"); 198 } 199 200 sqlite3_bind_text(stmt, 1, row[1].as(string()).c_str(), row[1].as(string()).length(), SQLITE_TRANSIENT); 201 sqlite3_bind_text(stmt, 2, row[0].as(string()).c_str(), row[0].as(string()).length(), SQLITE_TRANSIENT); 202 203 if (sqlite3_step(stmt) != SQLITE_DONE) 204 errx(1,"Could not execute query"); 205 sqlite3_reset(stmt); 206 sqlite3_finalize(stmt); 207 208 puts(ln.c_str()); 209 210 load_depends(depends_stmt, C2, runid, row[0].as(string()).c_str(), row[5].as(string()).c_str()); 211 } 212 } catch (const std::exception &e) { 213 fprintf(stderr, "Skipping port %s: %s\n", 214 row[0].as(string()).c_str(), e.what()); 215 free(filePath); 216 free(fileHash); 217 continue; 218 } 219 } 220 printf("\n"); 221 } else { 222 cerr << "Empty resultset" << endl; 223 } 224 225 printf("Load the mirrors list\n"); 226 result R2(N.exec("SELECT country, url FROM mirrors order by country")); 227 if (!R2.empty()) 228 { 229 for (result::const_iterator c = R2.begin(); c != R2.end(); ++c) 230 { 231 if (sqlite3_prepare_v2(db, "INSERT INTO mirrors (country, mirror) VALUES(?,?)", -1, &stmt, 0) != SQLITE_OK) 232 { 233 errx(1, "Could not prepare statement"); 234 } 235 string country = c[0].as(string()); 236 cout << country << endl; 237 string url = c[1].as(string()); 238 cout << url << endl; 239 sqlite3_bind_text(stmt, 1, country.c_str(), country.length(), SQLITE_TRANSIENT); 240 sqlite3_bind_text(stmt, 2, url.c_str(), url.length(), SQLITE_TRANSIENT); 241 242 if (sqlite3_step(stmt) != SQLITE_DONE) 243 errx(1, "Could not execute query"); 244 sqlite3_reset(stmt); 245 sqlite3_finalize(stmt); 246 } 247 } 248 249 printf("Load the MOVED list\n"); 250 snprintf(query_def, sizeof(query_def), "SELECT port, moved_to, why, date FROM moved where run=%d order by id", runid); 251 result R3(N.exec(string(query_def))); 252 if (!R3.empty()) 253 { 254 for (result::const_iterator c = R3.begin(); c != R3.end(); ++c) 255 { 256 if (sqlite3_prepare_v2(db, "INSERT INTO moved (port, moved_to, why, date) VALUES(?,?,?,?)", -1, &stmt, 0) != SQLITE_OK) 257 { 258 errx(1, "Could not prepare statement"); 259 } 260 string port = c[0].as(string()); 261 cout << port << endl; 262 string moved_to = c[1].as(string()); 263 cout << moved_to << endl; 264 string why = c[2].as(string()); 265 string date = c[3].as(string()); 266 sqlite3_bind_text(stmt, 1, port.c_str(), port.length(), SQLITE_TRANSIENT); 267 sqlite3_bind_text(stmt, 2, moved_to.c_str(), moved_to.length(), SQLITE_TRANSIENT); 268 sqlite3_bind_text(stmt, 3, why.c_str(), why.length(), SQLITE_TRANSIENT); 269 sqlite3_bind_text(stmt, 4, date.c_str(), date.length(), SQLITE_TRANSIENT); 270 271 if (sqlite3_step(stmt) != SQLITE_DONE) 272 errx(1, "Could not execute query"); 273 sqlite3_reset(stmt); 274 sqlite3_finalize(stmt); 275 } 276 } 277 278 printf("Load ALIASES not included in current run\n"); 279 snprintf(query_def, sizeof(query_def), "select distinct(pkgname, name, moved_to), pkgname, name, moved.moved_to from ports inner join runs on ports.run = runs.id left join moved on moved.port = ports.name where moved.run = %d and ports.run < %d and moved.port not in (SELECT name from ports where run = %d) and runs.arch = (select arch from runs where id = %d) group by pkgname, name, moved_to order by pkgname, name, moved_to;", runid, runid, runid, runid); 280 result R4(N.exec(string(query_def))); 281 if (!R4.empty()) 282 { 283 for (result::const_iterator c = R4.begin(); c != R4.end(); ++c) 284 { 285 if (sqlite3_prepare_v2(db, "INSERT INTO aliases (alias, pkg) VALUES(?,?)", -1, &stmt, 0) != SQLITE_OK) 286 { 287 errx(1, "Could not prepare statement"); 288 } 289 290 sqlite3_bind_text(stmt, 1, c[2].as(string()).c_str(), c[2].as(string()).length(), SQLITE_TRANSIENT); 291 sqlite3_bind_text(stmt, 2, c[1].as(string()).c_str(), c[1].as(string()).length(), SQLITE_TRANSIENT); 292 293 if (sqlite3_step(stmt) != SQLITE_DONE) 294 errx(1, "Could not execute query"); 295 sqlite3_reset(stmt); 296 sqlite3_finalize(stmt); 297 } 298 } 299 300 if (db != NULL && depends_stmt != NULL) 301 { 302 sqlite3_finalize(depends_stmt); 303 exec_indexdb(db, "COMMIT"); 304 close_indexdb(db); 305 } 306 307 printf("Mark run blessed\n"); 308 N.exec0( 309 "UPDATE runs " 310 "SET blessed = true, status = 'complete' " 311 "WHERE id = " + 312 pqxx::to_string(runid)); 313 314 } catch (const std::exception &e) { 315 fprintf(stderr, "magus-bless failed: %s\n", e.what()); 316 return 1; 317 } 318 319 return 0; 320 } 321 322 MagusConfig 323 load_config(const char *path) 324 { 325 struct ucl_parser *parser; 326 ucl_object_t *root; 327 MagusConfig config; 328 329 parser = ucl_parser_new(0); 330 if (parser == NULL) 331 errx(1, "Could not create config parser"); 332 333 if (!ucl_parser_add_file(parser, path)) 334 errx(1, "Could not parse %s: %s", path, ucl_parser_get_error(parser)); 335 336 root = ucl_parser_get_object(parser); 337 if (root == NULL) 338 errx(1, "Could not load %s", path); 339 340 config.db_host = config_string(root, "DBHost", "localhost"); 341 config.db_name = config_string(root, "DBName", "magus"); 342 config.db_user = config_string(root, "DBUser", "magus"); 343 config.db_pass = config_string(root, "DBPass", ""); 344 345 ucl_object_unref(root); 346 ucl_parser_free(parser); 347 348 return config; 349 } 350 351 string 352 config_string(const ucl_object_t *root, const char *key, const char *default_value) 353 { 354 const ucl_object_t *obj; 355 const char *value; 356 357 obj = ucl_object_find_key(root, key); 358 if (obj == NULL) 359 return string(default_value); 360 361 value = ucl_object_tostring(obj); 362 if (value == NULL) 363 errx(1, "Config key %s must be a string", key); 364 365 return string(value); 366 } 367 368 sqlite3* 369 open_indexdb(int runid) 370 { 371 sqlite3 *db = NULL; 372 char *filename = NULL; 373 374 asprintf(&filename, "index.db"); 375 if (filename == NULL) 376 { 377 errx(1, "Could not malloc filename"); 378 } 379 unlink(filename); 380 if (sqlite3_open(filename, &db) != SQLITE_OK) { 381 errx(1, "Could not open index db: %s", sqlite3_errmsg(db)); 382 } 383 free(filename); 384 return db; 385 } 386 387 void 388 close_indexdb(sqlite3 *db) 389 { 390 sqlite3_exec(db, "VACUUM", 0, 0, 0); 391 sqlite3_close(db); 392 } 393 394 395 int 396 exec_indexdb(sqlite3 *db, const char *fmt, ...) 397 { 398 va_list args; 399 char *sql; 400 int sqlcode; 401 402 va_start(args, fmt); 403 404 sql = sqlite3_vmprintf(fmt, args); 405 406 va_end(args); 407 408 if (sql == NULL) 409 errx(1,"Couldn't allocate memory for sql statement"); 410 411 sqlcode = sqlite3_exec(db, sql, 0, 0, 0); 412 /* if we get an error code, we want to run it again in some cases */ 413 if (sqlcode == SQLITE_BUSY || sqlcode == SQLITE_LOCKED) { 414 if (sqlite3_exec(db, sql, 0, 0, 0) != SQLITE_OK) { 415 sqlite3_free(sql); 416 errx(1, sqlite3_errmsg(db)); 417 } 418 } else if (sqlcode != SQLITE_OK) { 419 sqlite3_free(sql); 420 errx(1, sqlite3_errmsg(db)); 421 } 422 423 sqlite3_free(sql); 424 425 return 0; 426 } 427 428 void 429 create_indexdb(sqlite3 *db) 430 { 431 exec_indexdb(db, "CREATE TABLE IF NOT EXISTS mirrors (country text NOT NULL, mirror text NOT NULL)"); 432 exec_indexdb(db, "CREATE INDEX mirrors_country on mirrors(country)"); 433 exec_indexdb(db, "CREATE TABLE IF NOT EXISTS packages (pkg text NOT NULL, version text NOT NULL, license text NOT NULL, comment text NOT NULL, bundlefile text NOT NULL, hash text NOT NULL, type int NOT NULL)"); 434 exec_indexdb(db, "CREATE INDEX packages_pkg ON packages (pkg)"); /* should be unique */ 435 exec_indexdb(db, "CREATE TABLE IF NOT EXISTS aliases (alias text NOT NULL, pkg text NOT NULL)"); 436 exec_indexdb(db, "CREATE TABLE IF NOT EXISTS depends (pkg text NOT NULL, version text NOT NULL, d_pkg text NOT NULL, d_version text NOT NULL)"); 437 exec_indexdb(db, "CREATE TABLE IF NOT EXISTS moved (port text NOT NULL, moved_to text, why text, date text)"); 438 } 439 440 void 441 load_depends(sqlite3_stmt *stmt, connection & C, int runid, const char *pkg_name, const char *version) 442 { 443 printf("---->\tProcessing dependencies for %s - %s\n", pkg_name, version); 444 445 if (C.is_open()) 446 { 447 nontransaction N(C); 448 result R(N.exec( 449 "SELECT distinct p2.pkgname, p2.version from ports as p1 " 450 "left join depends d on p1.id = d.port " 451 "left join ports p2 on d.dependency = p2.id " 452 "where p2.run = $1 and p1.run = $2 " 453 "and ((p1.status = 'pass' or p1.status = 'warn') " 454 "and (p2.status = 'pass' or p2.status = 'warn')) " 455 "and p1.pkgname = $3 and p1.version = $4 " 456 "and d.type in ('run','lib','pkg')", 457 pqxx::params{runid, runid, pkg_name, version})); 458 459 if (!R.empty()) 460 { 461 for (result::const_iterator row = R.begin(); row != R.end(); ++row) 462 { 463 sqlite3_bind_text(stmt, 1, pkg_name, strlen(pkg_name), SQLITE_TRANSIENT); 464 sqlite3_bind_text(stmt, 2, version, strlen(version), SQLITE_TRANSIENT); 465 sqlite3_bind_text(stmt, 3, row[0].as(string()).c_str(), row[0].as(string()).length(), SQLITE_TRANSIENT); 466 sqlite3_bind_text(stmt, 4, row[1].as(string()).c_str(), row[1].as(string()).length(), SQLITE_TRANSIENT); 467 468 if (sqlite3_step(stmt) != SQLITE_DONE) 469 errx(1,"Could not execute query"); 470 sqlite3_reset(stmt); 471 sqlite3_clear_bindings(stmt); 472 } 473 } 474 } 475 } 476