1 /*- 2 Copyright (C) 2008, 2010, 2011, 2013, 2014, 2022 Lucas Holt All rights reserved. 3 4 Redistribution and use in source and binary forms, with or without 5 modification, are permitted provided that the following conditions 6 are met: 7 1. Redistributions of source code must retain the above copyright 8 notice, this list of conditions and the following disclaimer. 9 2. Redistributions in binary form must reproduce the above copyright 10 notice, this list of conditions and the following disclaimer in the 11 documentation and/or other materials provided with the distribution. 12 13 THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND 14 ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16 ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE 17 FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18 DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19 OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20 HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21 LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22 OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23 SUCH DAMAGE. 24 */ 25 26 #include <iostream> 27 #include <pqxx/pqxx> 28 #include <cstring> 29 30 using namespace std; 31 using namespace pqxx; 32 33 #include <stdio.h> 34 #include <stdlib.h> 35 #include <unistd.h> 36 #include <err.h> 37 38 #include <sys/types.h> 39 #include <sha256.h> 40 41 #include "sqlite3.h" 42 43 const string DB_HOST = "10.1.10.11"; 44 const string DB_DATABASE = "magus"; 45 46 /* SQLITE3 */ 47 sqlite3* open_indexdb(int); 48 void close_indexdb(sqlite3 *); 49 int exec_indexdb(sqlite3 *, const char *, ...); 50 void create_indexdb(sqlite3 *); 51 void load_depends(sqlite3 *, connection &, int, const char *, const char *); 52 53 int 54 main(int argc, char *argv[]) 55 { 56 char query_def[2048]; 57 int runid; 58 sqlite3 *db = NULL; 59 sqlite3_stmt *stmt; 60 char *fileHash; 61 char *filePath; 62 63 if (argc != 5) 64 { 65 fprintf( stderr, "Usage: %s <run id> <pg_user> <pg_pass> <files>\n", argv[0] ); 66 exit(1); 67 } 68 69 runid = atoi(argv[1]); 70 if (runid < 1) 71 { 72 fprintf( stderr, "Invalid run id %d\n", runid); 73 exit(1); 74 } 75 76 string connect_string = "dbname=magus user=" + string(argv[2]) + " password=" + string(argv[3]) + " hostaddr=" + DB_HOST + " port=5432"; 77 connection C(connect_string); 78 connection C2(connect_string); 79 80 if (C.is_open()) { 81 cout << "We are connected to " << C.dbname() << endl; 82 } else { 83 cout << "We are not connected! Check username and password." << endl; 84 return -1; 85 } 86 87 if (!C2.is_open()) { 88 cout << "Secondary connection failed! Check username and password." << endl; 89 return -1; 90 } 91 92 snprintf(query_def, sizeof(query_def), 93 "select pkgname, name, license, description, CONCAT(CONCAT_WS( '-', pkgname, version),'.mport'), version, restricted from ports where run=%d AND status!='internal' AND status!='untested' AND status!='fail' ORDER BY pkgname;", runid); 94 95 nontransaction N(C); 96 97 result R(N.exec(string(query_def))); 98 99 if (!R.empty()) 100 { 101 102 cout << "Init index db file" << endl; 103 db = open_indexdb(runid); 104 create_indexdb(db); 105 106 for (result::const_iterator row = R.begin(); row != R.end(); ++row) 107 { 108 109 string ln = row[0].as(string()) + ": " + row[1].as(string()) + " " + row[2].as(string()) + " " + row[3].as(string()) + " " + row[5].as(string()) + " " + row[4].as(string()); 110 if (asprintf(&filePath, "%s/%s", argv[4], row[4].as(string()).c_str()) == -1) 111 errx(1, "Could not allocate file path"); 112 fileHash = SHA256_File(filePath, NULL); 113 if (fileHash == NULL) 114 { 115 fprintf(stderr, "Could not locate file %s\n", filePath); 116 free(filePath); 117 continue; 118 } 119 120 if (row[6].as(bool())) 121 { 122 fprintf(stderr, "File %s is restricted and will be removed.\n", filePath); 123 unlink(filePath); 124 free(fileHash); 125 free(filePath); 126 continue; 127 } 128 129 { 130 if (sqlite3_prepare_v2(db, 131 "INSERT INTO packages (pkg, version, license, comment, bundlefile, hash, type) VALUES(?,?,?,?,?,?,?)", 132 -1, &stmt, 0) != SQLITE_OK) 133 { 134 errx(1, "Could not prepare statement"); 135 } 136 sqlite3_bind_text(stmt, 1, row[0].as(string()).c_str(), row[0].as(string()).length(), SQLITE_TRANSIENT); 137 sqlite3_bind_text(stmt, 2, row[5].as(string()).c_str(), row[5].as(string()).length(), SQLITE_TRANSIENT); 138 sqlite3_bind_text(stmt, 3, row[2].as(string()).c_str(), row[2].as(string()).length(), SQLITE_TRANSIENT); 139 sqlite3_bind_text(stmt, 4, row[3].as(string()).c_str(), row[3].as(string()).length(), SQLITE_TRANSIENT); 140 sqlite3_bind_text(stmt, 5, row[4].as(string()).c_str(), row[4].as(string()).length(), SQLITE_TRANSIENT); 141 sqlite3_bind_text(stmt, 6, fileHash, strlen(fileHash), SQLITE_TRANSIENT); 142 sqlite3_bind_int(stmt, 7, 0); // TODO: handle system packages? 143 144 if (sqlite3_step(stmt) != SQLITE_DONE) 145 errx(1,"Could not execute query"); 146 sqlite3_reset(stmt); 147 sqlite3_finalize(stmt); 148 free(filePath); 149 free(fileHash); 150 151 if (sqlite3_prepare_v2(db, "INSERT INTO aliases (alias, pkg) VALUES(?,?)", -1, &stmt, 0) != SQLITE_OK) 152 { 153 errx(1, "Could not prepare statement"); 154 } 155 156 sqlite3_bind_text(stmt, 1, row[1].as(string()).c_str(), row[1].as(string()).length(), SQLITE_TRANSIENT); 157 sqlite3_bind_text(stmt, 2, row[0].as(string()).c_str(), row[0].as(string()).length(), SQLITE_TRANSIENT); 158 159 if (sqlite3_step(stmt) != SQLITE_DONE) 160 errx(1,"Could not execute query"); 161 sqlite3_reset(stmt); 162 sqlite3_finalize(stmt); 163 164 puts(ln.c_str()); 165 166 load_depends(db, C2, runid, row[0].as(string()).c_str(), row[5].as(string()).c_str()); 167 } 168 } 169 printf("\n"); 170 } else { 171 cerr << "Empty resultset" << endl; 172 } 173 174 printf("Load the mirrors list\n"); 175 result R2(N.exec("SELECT country, url FROM mirrors order by country")); 176 if (!R2.empty()) 177 { 178 for (result::const_iterator c = R2.begin(); c != R2.end(); ++c) 179 { 180 if (sqlite3_prepare_v2(db, "INSERT INTO mirrors (country, mirror) VALUES(?,?)", -1, &stmt, 0) != SQLITE_OK) 181 { 182 errx(1, "Could not prepare statement"); 183 } 184 string country = c[0].as(string()); 185 cout << country << endl; 186 string url = c[1].as(string()); 187 cout << url << endl; 188 sqlite3_bind_text(stmt, 1, country.c_str(), country.length(), SQLITE_TRANSIENT); 189 sqlite3_bind_text(stmt, 2, url.c_str(), url.length(), SQLITE_TRANSIENT); 190 191 if (sqlite3_step(stmt) != SQLITE_DONE) 192 errx(1, "Could not execute query"); 193 sqlite3_reset(stmt); 194 sqlite3_finalize(stmt); 195 } 196 } 197 198 printf("Load the MOVED list\n"); 199 snprintf(query_def, sizeof(query_def), "SELECT port, moved_to, why, date FROM moved where run=%d order by id", runid); 200 result R3(N.exec(string(query_def))); 201 if (!R3.empty()) 202 { 203 for (result::const_iterator c = R3.begin(); c != R3.end(); ++c) 204 { 205 if (sqlite3_prepare_v2(db, "INSERT INTO moved (port, moved_to, why, date) VALUES(?,?,?,?)", -1, &stmt, 0) != SQLITE_OK) 206 { 207 errx(1, "Could not prepare statement"); 208 } 209 string port = c[0].as(string()); 210 cout << port << endl; 211 string moved_to = c[1].as(string()); 212 cout << moved_to << endl; 213 string why = c[2].as(string()); 214 string date = c[3].as(string()); 215 sqlite3_bind_text(stmt, 1, port.c_str(), port.length(), SQLITE_TRANSIENT); 216 sqlite3_bind_text(stmt, 2, moved_to.c_str(), moved_to.length(), SQLITE_TRANSIENT); 217 sqlite3_bind_text(stmt, 3, why.c_str(), why.length(), SQLITE_TRANSIENT); 218 sqlite3_bind_text(stmt, 4, date.c_str(), date.length(), SQLITE_TRANSIENT); 219 220 if (sqlite3_step(stmt) != SQLITE_DONE) 221 errx(1, "Could not execute query"); 222 sqlite3_reset(stmt); 223 sqlite3_finalize(stmt); 224 } 225 } 226 227 printf("Load ALIASES not included in current run\n"); 228 snprintf(query_def, sizeof(query_def), "select distinct(pkgname, name, moved_to), pkgname, name, moved.moved_to from ports inner join runs on ports.run = runs.id left join moved on moved.port = ports.name where moved.run = %d and ports.run < %d and moved.port not in (SELECT name from ports where run = %d) and runs.arch = (select arch from runs where id = %d) group by pkgname, name, moved_to order by pkgname, name, moved_to;", runid, runid, runid, runid); 229 result R4(N.exec(string(query_def))); 230 if (!R4.empty()) 231 { 232 for (result::const_iterator c = R4.begin(); c != R4.end(); ++c) 233 { 234 if (sqlite3_prepare_v2(db, "INSERT INTO aliases (alias, pkg) VALUES(?,?)", -1, &stmt, 0) != SQLITE_OK) 235 { 236 errx(1, "Could not prepare statement"); 237 } 238 239 sqlite3_bind_text(stmt, 1, c[2].as(string()).c_str(), c[2].as(string()).length(), SQLITE_TRANSIENT); 240 sqlite3_bind_text(stmt, 2, c[1].as(string()).c_str(), c[1].as(string()).length(), SQLITE_TRANSIENT); 241 242 if (sqlite3_step(stmt) != SQLITE_DONE) 243 errx(1, "Could not execute query"); 244 sqlite3_reset(stmt); 245 sqlite3_finalize(stmt); 246 } 247 } 248 249 close_indexdb(db); 250 251 printf("Mark run blessed\n"); 252 N.exec0( 253 "UPDATE runs " 254 "SET blessed = true, status = 'complete' " 255 "WHERE id = " + 256 pqxx::to_string(runid)); 257 258 return 0; 259 } 260 261 sqlite3* 262 open_indexdb(int runid) 263 { 264 sqlite3 *db = NULL; 265 char *filename = NULL; 266 267 asprintf(&filename, "index.db"); 268 if (filename == NULL) 269 { 270 errx(1, "Could not malloc filename"); 271 } 272 unlink(filename); 273 if (sqlite3_open(filename, &db) != SQLITE_OK) { 274 errx(1, "Could not open index db: %s", sqlite3_errmsg(db)); 275 } 276 free(filename); 277 return db; 278 } 279 280 void 281 close_indexdb(sqlite3 *db) 282 { 283 sqlite3_exec(db, "VACUUM", 0, 0, 0); 284 sqlite3_close(db); 285 } 286 287 288 int 289 exec_indexdb(sqlite3 *db, const char *fmt, ...) 290 { 291 va_list args; 292 char *sql; 293 int sqlcode; 294 295 va_start(args, fmt); 296 297 sql = sqlite3_vmprintf(fmt, args); 298 299 va_end(args); 300 301 if (sql == NULL) 302 errx(1,"Couldn't allocate memory for sql statement"); 303 304 sqlcode = sqlite3_exec(db, sql, 0, 0, 0); 305 /* if we get an error code, we want to run it again in some cases */ 306 if (sqlcode == SQLITE_BUSY || sqlcode == SQLITE_LOCKED) { 307 if (sqlite3_exec(db, sql, 0, 0, 0) != SQLITE_OK) { 308 sqlite3_free(sql); 309 errx(1, sqlite3_errmsg(db)); 310 } 311 } else if (sqlcode != SQLITE_OK) { 312 sqlite3_free(sql); 313 errx(1, sqlite3_errmsg(db)); 314 } 315 316 sqlite3_free(sql); 317 318 return 0; 319 } 320 321 void 322 create_indexdb(sqlite3 *db) 323 { 324 exec_indexdb(db, "CREATE TABLE IF NOT EXISTS mirrors (country text NOT NULL, mirror text NOT NULL)"); 325 exec_indexdb(db, "CREATE INDEX mirrors_country on mirrors(country)"); 326 exec_indexdb(db, "CREATE TABLE IF NOT EXISTS packages (pkg text NOT NULL, version text NOT NULL, license text NOT NULL, comment text NOT NULL, bundlefile text NOT NULL, hash text NOT NULL, type int NOT NULL)"); 327 exec_indexdb(db, "CREATE INDEX packages_pkg ON packages (pkg)"); /* should be unique */ 328 exec_indexdb(db, "CREATE TABLE IF NOT EXISTS aliases (alias text NOT NULL, pkg text NOT NULL)"); 329 exec_indexdb(db, "CREATE TABLE IF NOT EXISTS depends (pkg text NOT NULL, version text NOT NULL, d_pkg text NOT NULL, d_version text NOT NULL)"); 330 exec_indexdb(db, "CREATE TABLE IF NOT EXISTS moved (port text NOT NULL, moved_to text, why text, date text)"); 331 } 332 333 void 334 load_depends(sqlite3 *db, connection & C, int runid, const char *pkg_name, const char *version) 335 { 336 char query_def[2048]; 337 sqlite3_stmt *stmt; 338 339 printf("---->\tProcessing dependencies for %s - %s\n", pkg_name, version); 340 341 snprintf(query_def, sizeof(query_def), "SELECT distinct p2.pkgname, p2.version from ports as p1 left join depends d on p1.id = d.port left join ports p2 on d.dependency = p2.id where p2.run = %d and p1.run = %d and ((p1.status = 'pass' or p1.status = 'warn') and (p2.status = 'pass' or p2.status = 'warn')) and p1.pkgname = '%s' and p1.version = '%s' and d.type in ('run','lib', 'pkg')", 342 runid, runid, pkg_name, version); 343 344 if (C.is_open()) 345 { 346 nontransaction N(C); 347 result R(N.exec(string(query_def))); 348 349 if (!R.empty()) 350 { 351 for (result::const_iterator row = R.begin(); row != R.end(); ++row) 352 { 353 if (sqlite3_prepare_v2(db, 354 "INSERT INTO depends (pkg, version, d_pkg, d_version) VALUES(?,?,?,?)", 355 -1, &stmt, 0) != SQLITE_OK) 356 { 357 errx(1, "Could not prepare statement"); 358 } 359 sqlite3_bind_text(stmt, 1, pkg_name, strlen(pkg_name), SQLITE_TRANSIENT); 360 sqlite3_bind_text(stmt, 2, version, strlen(version), SQLITE_TRANSIENT); 361 sqlite3_bind_text(stmt, 3, row[0].as(string()).c_str(), row[0].as(string()).length(), SQLITE_TRANSIENT); 362 sqlite3_bind_text(stmt, 4, row[1].as(string()).c_str(), row[1].as(string()).length(), SQLITE_TRANSIENT); 363 364 if (sqlite3_step(stmt) != SQLITE_DONE) 365 errx(1,"Could not execute query"); 366 sqlite3_reset(stmt); 367 sqlite3_finalize(stmt); 368 } 369 } 370 } 371 } 372