1 /* $NetBSD: apropos-utils.c,v 1.51 2023/08/03 07:49:23 rin Exp $ */
2 /*-
3 * Copyright (c) 2011 Abhinav Upadhyay <er.abhinav.upadhyay@gmail.com>
4 * All rights reserved.
5 *
6 * This code was developed as part of Google's Summer of Code 2011 program.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 *
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in
16 * the documentation and/or other materials provided with the
17 * distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
22 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
23 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
24 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
25 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
26 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
27 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
28 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
29 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
30 * SUCH DAMAGE.
31 */
32
33 #include <sys/cdefs.h>
34 __RCSID("$NetBSD: apropos-utils.c,v 1.51 2023/08/03 07:49:23 rin Exp $");
35
36 #include <sys/queue.h>
37 #include <sys/stat.h>
38
39 #include <assert.h>
40 #include <ctype.h>
41 #include <err.h>
42 #include <math.h>
43 #include <stdio.h>
44 #include <stdlib.h>
45 #include <string.h>
46 #include <util.h>
47 #include <zlib.h>
48 #include <term.h>
49 #include <unistd.h>
50 #undef tab // XXX: manconf.h
51
52 #include "apropos-utils.h"
53 #include "custom_apropos_tokenizer.h"
54 #include "manconf.h"
55 #include "fts3_tokenizer.h"
56
57 typedef struct orig_callback_data {
58 void *data;
59 int (*callback) (query_callback_args*);
60 } orig_callback_data;
61
62 typedef struct inverse_document_frequency {
63 double value;
64 int status;
65 } inverse_document_frequency;
66
67 /* weights for individual columns */
68 static const double col_weights[] = {
69 2.0, // NAME
70 2.00, // Name-description
71 0.55, // DESCRIPTION
72 0.10, // LIBRARY
73 0.001, //RETURN VALUES
74 0.20, //ENVIRONMENT
75 0.01, //FILES
76 0.001, //EXIT STATUS
77 2.00, //DIAGNOSTICS
78 0.05, //ERRORS
79 0.00, //md5_hash
80 1.00 //machine
81 };
82
83 #ifndef APROPOS_DEBUG
84 static int
register_tokenizer(sqlite3 * db)85 register_tokenizer(sqlite3 *db)
86 {
87 int rc;
88 sqlite3_stmt *stmt;
89 const sqlite3_tokenizer_module *p;
90 const char *name = "custom_apropos_tokenizer";
91 get_custom_apropos_tokenizer(&p);
92 const char *sql = "SELECT fts3_tokenizer(?, ?)";
93
94 sqlite3_db_config(db, SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER, 1, 0);
95 rc = sqlite3_prepare_v2(db, sql, -1, &stmt, 0);
96 if (rc != SQLITE_OK)
97 return rc;
98
99 sqlite3_bind_text(stmt, 1, name, -1, SQLITE_STATIC);
100 sqlite3_bind_blob(stmt, 2, &p, sizeof(p), SQLITE_STATIC);
101 sqlite3_step(stmt);
102
103 return sqlite3_finalize(stmt);
104 }
105 #endif
106
107 /*
108 * lower --
109 * Converts the string str to lower case
110 */
111 char *
lower(char * str)112 lower(char *str)
113 {
114 assert(str);
115 int i = 0;
116 char c;
117 while ((c = str[i]) != '\0')
118 str[i++] = tolower((unsigned char) c);
119 return str;
120 }
121
122 /*
123 * concat--
124 * Utility function. Concatenates together: dst, a space character and src.
125 * dst + " " + src
126 */
127 void
concat(char ** dst,const char * src)128 concat(char **dst, const char *src)
129 {
130 concat2(dst, src, strlen(src));
131 }
132
133 void
concat2(char ** dst,const char * src,size_t srclen)134 concat2(char **dst, const char *src, size_t srclen)
135 {
136 size_t totallen, dstlen;
137 char *mydst = *dst;
138 assert(src != NULL);
139
140 /*
141 * If destination buffer dst is NULL, then simply
142 * strdup the source buffer
143 */
144 if (mydst == NULL) {
145 mydst = estrndup(src, srclen);
146 *dst = mydst;
147 return;
148 }
149
150 dstlen = strlen(mydst);
151 /*
152 * NUL Byte and separator space
153 */
154 totallen = dstlen + srclen + 2;
155
156 mydst = erealloc(mydst, totallen);
157
158 /* Append a space at the end of dst */
159 mydst[dstlen++] = ' ';
160
161 /* Now, copy src at the end of dst */
162 memcpy(mydst + dstlen, src, srclen);
163 mydst[dstlen + srclen] = '\0';
164 *dst = mydst;
165 }
166
167 void
close_db(sqlite3 * db)168 close_db(sqlite3 *db)
169 {
170 sqlite3_close(db);
171 sqlite3_shutdown();
172 }
173
174 /*
175 * create_db --
176 * Creates the database schema.
177 */
178 static int
create_db(sqlite3 * db)179 create_db(sqlite3 *db)
180 {
181 const char *sqlstr = NULL;
182 char *schemasql;
183 char *errmsg = NULL;
184
185 /*------------------------ Create the tables------------------------------*/
186
187 #if NOTYET
188 sqlite3_exec(db, "PRAGMA journal_mode = WAL", NULL, NULL, NULL);
189 #else
190 sqlite3_exec(db, "PRAGMA journal_mode = DELETE", NULL, NULL, NULL);
191 #endif
192
193 schemasql = sqlite3_mprintf("PRAGMA user_version = %d",
194 APROPOS_SCHEMA_VERSION);
195 sqlite3_exec(db, schemasql, NULL, NULL, &errmsg);
196 if (errmsg != NULL)
197 goto out;
198 sqlite3_free(schemasql);
199
200 sqlstr =
201 //mandb
202 "CREATE VIRTUAL TABLE mandb USING fts4(section, name, "
203 "name_desc, desc, lib, return_vals, env, files, "
204 "exit_status, diagnostics, errors, md5_hash UNIQUE, machine, "
205 #ifndef APROPOS_DEBUG
206 "compress=zip, uncompress=unzip, tokenize=custom_apropos_tokenizer, "
207 #else
208 "tokenize=porter, "
209 #endif
210 "notindexed=section, notindexed=md5_hash); "
211 //mandb_meta
212 "CREATE TABLE IF NOT EXISTS mandb_meta(device, inode, mtime, "
213 "file UNIQUE, md5_hash UNIQUE, id INTEGER PRIMARY KEY); "
214 //mandb_links
215 "CREATE TABLE IF NOT EXISTS mandb_links(link COLLATE NOCASE, target, section, "
216 "machine, md5_hash, name_desc); ";
217
218 sqlite3_exec(db, sqlstr, NULL, NULL, &errmsg);
219 if (errmsg != NULL)
220 goto out;
221
222 sqlstr =
223 "CREATE INDEX IF NOT EXISTS index_mandb_links ON mandb_links "
224 "(link); "
225 "CREATE INDEX IF NOT EXISTS index_mandb_meta_dev ON mandb_meta "
226 "(device, inode); "
227 "CREATE INDEX IF NOT EXISTS index_mandb_links_md5 ON mandb_links "
228 "(md5_hash);";
229 sqlite3_exec(db, sqlstr, NULL, NULL, &errmsg);
230 if (errmsg != NULL)
231 goto out;
232 return 0;
233
234 out:
235 warnx("%s", errmsg);
236 free(errmsg);
237 sqlite3_close(db);
238 sqlite3_shutdown();
239 return -1;
240 }
241
242 /*
243 * zip --
244 * User defined SQLite function to compress the FTS table
245 */
246 static void
zip(sqlite3_context * pctx,int nval,sqlite3_value ** apval)247 zip(sqlite3_context *pctx, int nval, sqlite3_value **apval)
248 {
249 int nin;
250 long int nout;
251 const unsigned char * inbuf;
252 unsigned char *outbuf;
253
254 assert(nval == 1);
255 nin = sqlite3_value_bytes(apval[0]);
256 inbuf = (const unsigned char *) sqlite3_value_blob(apval[0]);
257 nout = nin + 13 + (nin + 999) / 1000;
258 outbuf = emalloc(nout);
259 compress(outbuf, (unsigned long *) &nout, inbuf, nin);
260 sqlite3_result_blob(pctx, outbuf, nout, free);
261 }
262
263 /*
264 * unzip --
265 * User defined SQLite function to uncompress the FTS table.
266 */
267 static void
unzip(sqlite3_context * pctx,int nval,sqlite3_value ** apval)268 unzip(sqlite3_context *pctx, int nval, sqlite3_value **apval)
269 {
270 unsigned int rc;
271 unsigned char *outbuf;
272 z_stream stream;
273 long total_out;
274
275 assert(nval == 1);
276 memset(&stream, 0, sizeof(stream));
277 stream.next_in = __UNCONST(sqlite3_value_blob(apval[0]));
278 stream.avail_in = sqlite3_value_bytes(apval[0]);
279 stream.zalloc = NULL;
280 stream.zfree = NULL;
281
282 if (inflateInit(&stream) != Z_OK) {
283 return;
284 }
285
286 total_out = stream.avail_out = stream.avail_in * 2 + 100;
287 stream.next_out = outbuf = emalloc(stream.avail_out);
288 while ((rc = inflate(&stream, Z_SYNC_FLUSH)) != Z_STREAM_END) {
289 if (rc != Z_OK ||
290 (stream.avail_out != 0 && stream.avail_in == 0)) {
291 free(outbuf);
292 return;
293 }
294 total_out <<= 1;
295 outbuf = erealloc(outbuf, total_out);
296 stream.next_out = outbuf + stream.total_out;
297 stream.avail_out = total_out - stream.total_out;
298 }
299 if (inflateEnd(&stream) != Z_OK) {
300 free(outbuf);
301 return;
302 }
303 if (stream.total_out == 0) {
304 free(outbuf);
305 return;
306 }
307 outbuf = erealloc(outbuf, stream.total_out);
308 sqlite3_result_text(pctx, (const char *)outbuf, stream.total_out, free);
309 }
310
311 /*
312 * get_dbpath --
313 * Read the path of the database from man.conf and return.
314 */
315 char *
get_dbpath(const char * manconf)316 get_dbpath(const char *manconf)
317 {
318 TAG *tp;
319 char *dbpath;
320
321 config(manconf);
322 tp = gettag("_mandb", 1);
323 if (!tp)
324 return NULL;
325
326 if (TAILQ_EMPTY(&tp->entrylist))
327 return NULL;
328
329 dbpath = TAILQ_LAST(&tp->entrylist, tqh)->s;
330 return dbpath;
331 }
332
333 /* init_db --
334 * Prepare the database. Register the compress/uncompress functions and the
335 * stopword tokenizer.
336 * db_flag specifies the mode in which to open the database. 3 options are
337 * available:
338 * 1. DB_READONLY: Open in READONLY mode. An error if db does not exist.
339 * 2. DB_READWRITE: Open in read-write mode. An error if db does not exist.
340 * 3. DB_CREATE: Open in read-write mode. It will try to create the db if
341 * it does not exist already.
342 * RETURN VALUES:
343 * The function will return NULL in case the db does not exist
344 * and DB_CREATE
345 * was not specified. And in case DB_CREATE was specified and yet NULL is
346 * returned, then there was some other error.
347 * In normal cases the function should return a handle to the db.
348 */
349 sqlite3 *
init_db(mandb_access_mode db_flag,const char * manconf)350 init_db(mandb_access_mode db_flag, const char *manconf)
351 {
352 sqlite3 *db = NULL;
353 sqlite3_stmt *stmt;
354 struct stat sb;
355 int rc;
356 int create_db_flag = 0;
357
358 char *dbpath = get_dbpath(manconf);
359 if (dbpath == NULL)
360 errx(EXIT_FAILURE, "_mandb entry not found in man.conf");
361
362 if (!(stat(dbpath, &sb) == 0 && S_ISREG(sb.st_mode))) {
363 /* Database does not exist, check if DB_CREATE was specified,
364 * and set flag to create the database schema
365 */
366 if (db_flag != (MANDB_CREATE)) {
367 warnx("Missing apropos database. "
368 "Please run makemandb to create it.");
369 return NULL;
370 }
371 create_db_flag = 1;
372 } else {
373 /*
374 * Database exists. Check if we have the permissions
375 * to read/write the files
376 */
377 int access_mode = R_OK;
378 switch (db_flag) {
379 case MANDB_CREATE:
380 case MANDB_WRITE:
381 access_mode |= W_OK;
382 break;
383 default:
384 break;
385 }
386 if ((access(dbpath, access_mode)) != 0) {
387 warnx("Unable to access the database, please check"
388 " permissions for `%s'", dbpath);
389 return NULL;
390 }
391 }
392
393 sqlite3_initialize();
394 rc = sqlite3_open_v2(dbpath, &db, db_flag, NULL);
395
396 if (rc != SQLITE_OK) {
397 warnx("%s", sqlite3_errmsg(db));
398 goto error;
399 }
400
401 sqlite3_extended_result_codes(db, 1);
402
403 #ifndef APROPOS_DEBUG
404 rc = register_tokenizer(db);
405 if (rc != SQLITE_OK) {
406 warnx("Unable to register custom tokenizer: %s", sqlite3_errmsg(db));
407 goto error;
408 }
409 #endif
410
411 if (create_db_flag && create_db(db) < 0) {
412 warnx("%s", "Unable to create database schema");
413 goto error;
414 }
415
416 rc = sqlite3_prepare_v2(db, "PRAGMA user_version", -1, &stmt, NULL);
417 if (rc != SQLITE_OK) {
418 warnx("Unable to query schema version: %s",
419 sqlite3_errmsg(db));
420 goto error;
421 }
422 if (sqlite3_step(stmt) != SQLITE_ROW) {
423 sqlite3_finalize(stmt);
424 warnx("Unable to query schema version: %s",
425 sqlite3_errmsg(db));
426 goto error;
427 }
428 if (sqlite3_column_int(stmt, 0) != APROPOS_SCHEMA_VERSION) {
429 sqlite3_finalize(stmt);
430 warnx("Incorrect schema version found. "
431 "Please run makemandb -f.");
432 goto error;
433 }
434 sqlite3_finalize(stmt);
435
436
437 /* Register the zip and unzip functions for FTS compression */
438 rc = sqlite3_create_function(db, "zip", 1, SQLITE_ANY, NULL, zip,
439 NULL, NULL);
440 if (rc != SQLITE_OK) {
441 warnx("Unable to register function: compress: %s",
442 sqlite3_errmsg(db));
443 goto error;
444 }
445
446 rc = sqlite3_create_function(db, "unzip", 1, SQLITE_ANY, NULL,
447 unzip, NULL, NULL);
448 if (rc != SQLITE_OK) {
449 warnx("Unable to register function: uncompress: %s",
450 sqlite3_errmsg(db));
451 goto error;
452 }
453 return db;
454
455 error:
456 close_db(db);
457 return NULL;
458 }
459
460 /*
461 * rank_func --
462 * SQLite user defined function for ranking the documents.
463 * For each phrase of the query, it computes the tf and idf and adds them over.
464 * It computes the final rank, by multiplying tf and idf together.
465 * Weight of term t for document d = (term frequency of t in d *
466 * inverse document frequency of t)
467 *
468 * Term Frequency of term t in document d = Number of times t occurs in d /
469 * Number of times t appears in all documents
470 *
471 * Inverse document frequency of t = log(Total number of documents /
472 * Number of documents in which t occurs)
473 */
474 static void
rank_func(sqlite3_context * pctx,int nval,sqlite3_value ** apval)475 rank_func(sqlite3_context *pctx, int nval, sqlite3_value **apval)
476 {
477 inverse_document_frequency *idf = sqlite3_user_data(pctx);
478 double tf = 0.0;
479 const unsigned int *matchinfo;
480 int ncol;
481 int nphrase;
482 int iphrase;
483 int ndoc;
484 int doclen = 0;
485 const double k = 3.75;
486 /*
487 * Check that the number of arguments passed to this
488 * function is correct.
489 */
490 assert(nval == 1);
491
492 matchinfo = (const unsigned int *) sqlite3_value_blob(apval[0]);
493 nphrase = matchinfo[0];
494 ncol = matchinfo[1];
495 ndoc = matchinfo[2 + 3 * ncol * nphrase + ncol];
496 for (iphrase = 0; iphrase < nphrase; iphrase++) {
497 int icol;
498 const unsigned int *phraseinfo =
499 &matchinfo[2 + ncol + iphrase * ncol * 3];
500 for(icol = 1; icol < ncol; icol++) {
501
502 /* nhitcount: number of times the current phrase occurs
503 * in the current column in the current document.
504 * nglobalhitcount: number of times current phrase
505 * occurs in the current column in all documents.
506 * ndocshitcount: number of documents in which the
507 * current phrase occurs in the current column at
508 * least once.
509 */
510 int nhitcount = phraseinfo[3 * icol];
511 int nglobalhitcount = phraseinfo[3 * icol + 1];
512 int ndocshitcount = phraseinfo[3 * icol + 2];
513 doclen = matchinfo[2 + icol ];
514 double weight = col_weights[icol - 1];
515 if (idf->status == 0 && ndocshitcount)
516 idf->value +=
517 log(((double)ndoc / ndocshitcount))* weight;
518
519 /*
520 * Dividing the tf by document length to normalize
521 * the effect of longer documents.
522 */
523 if (nglobalhitcount > 0 && nhitcount)
524 tf += (((double)nhitcount * weight)
525 / (nglobalhitcount * doclen));
526 }
527 }
528 idf->status = 1;
529
530 /*
531 * Final score: Dividing by k + tf further normalizes the weight
532 * leading to better results. The value of k is experimental
533 */
534 double score = (tf * idf->value) / (k + tf);
535 sqlite3_result_double(pctx, score);
536 return;
537 }
538
539 /*
540 * generates sql query for matching the user entered query
541 */
542 static char *
generate_search_query(query_args * args,const char * snippet_args[3])543 generate_search_query(query_args *args, const char *snippet_args[3])
544 {
545 const char *default_snippet_args[3];
546 char *section_clause = NULL;
547 char *limit_clause = NULL;
548 char *machine_clause = NULL;
549 char *query = NULL;
550
551 if (args->machine) {
552 machine_clause = sqlite3_mprintf("AND mandb.machine=%Q", args->machine);
553 if (machine_clause == NULL)
554 goto RETURN;
555 }
556
557 if (args->nrec >= 0) {
558 /* Use the provided number of records and offset */
559 limit_clause = sqlite3_mprintf(" LIMIT %d OFFSET %d",
560 args->nrec, args->offset);
561 if (limit_clause == NULL)
562 goto RETURN;
563 }
564
565 /* We want to build a query of the form: "select x,y,z from mandb where
566 * mandb match :query [AND (section IN ('1', '2')]
567 * ORDER BY rank DESC [LIMIT 10 OFFSET 0]"
568 * NOTES:
569 * 1. The portion in first pair of square brackets is optional.
570 * It will be there only if the user has specified an option
571 * to search in one or more specific sections.
572 * 2. The LIMIT portion will be there if the user has specified
573 * a limit using the -n option.
574 */
575 if (args->sections && args->sections[0]) {
576 concat(§ion_clause, " AND mandb.section IN (");
577 for (size_t i = 0; args->sections[i]; i++) {
578 char *temp;
579 char c = args->sections[i + 1]? ',': ')';
580 if ((temp = sqlite3_mprintf("%Q%c", args->sections[i], c)) == NULL)
581 goto RETURN;
582 concat(§ion_clause, temp);
583 sqlite3_free(temp);
584 }
585 }
586
587 if (snippet_args == NULL) {
588 default_snippet_args[0] = "";
589 default_snippet_args[1] = "";
590 default_snippet_args[2] = "...";
591 snippet_args = default_snippet_args;
592 }
593
594 if (args->legacy) {
595 char *wild;
596 easprintf(&wild, "%%%s%%", args->search_str);
597 query = sqlite3_mprintf("SELECT section, name, name_desc, machine"
598 " FROM mandb"
599 " WHERE name LIKE %Q OR name_desc LIKE %Q "
600 "%s"
601 "%s",
602 wild, wild,
603 section_clause ? section_clause : "",
604 limit_clause ? limit_clause : "");
605 free(wild);
606 } else if (strchr(args->search_str, ' ') == NULL) {
607 /*
608 * If it's a single word query, we want to search in the
609 * links table as well. If the link table contains an entry
610 * for the queried keyword, we want to use that as the name of
611 * the man page.
612 * For example, for `apropos realloc` the output should be
613 * realloc(3) and not malloc(3).
614 */
615 query = sqlite3_mprintf(
616 "SELECT section, name, name_desc, machine,"
617 " snippet(mandb, %Q, %Q, %Q, -1, 40 ),"
618 " rank_func(matchinfo(mandb, \"pclxn\")) AS rank"
619 " FROM mandb WHERE name NOT IN ("
620 " SELECT target FROM mandb_links WHERE link=%Q AND"
621 " mandb_links.section=mandb.section) AND mandb MATCH %Q %s %s"
622 " UNION"
623 " SELECT mandb.section, mandb_links.link AS name, mandb.name_desc,"
624 " mandb.machine, '' AS snippet, 100.00 AS rank"
625 " FROM mandb JOIN mandb_links ON mandb.name=mandb_links.target and"
626 " mandb.section=mandb_links.section WHERE mandb_links.link=%Q"
627 " %s %s"
628 " ORDER BY rank DESC %s",
629 snippet_args[0], snippet_args[1], snippet_args[2],
630 args->search_str, args->search_str, section_clause ? section_clause : "",
631 machine_clause ? machine_clause : "", args->search_str,
632 machine_clause ? machine_clause : "",
633 section_clause ? section_clause : "",
634 limit_clause ? limit_clause : "");
635 } else {
636 query = sqlite3_mprintf("SELECT section, name, name_desc, machine,"
637 " snippet(mandb, %Q, %Q, %Q, -1, 40 ),"
638 " rank_func(matchinfo(mandb, \"pclxn\")) AS rank"
639 " FROM mandb"
640 " WHERE mandb MATCH %Q %s "
641 "%s"
642 " ORDER BY rank DESC"
643 "%s",
644 snippet_args[0], snippet_args[1], snippet_args[2],
645 args->search_str, machine_clause ? machine_clause : "",
646 section_clause ? section_clause : "",
647 limit_clause ? limit_clause : "");
648 }
649
650 RETURN:
651 sqlite3_free(machine_clause);
652 free(section_clause);
653 sqlite3_free(limit_clause);
654 return query;
655 }
656
657 static const char *
get_stmt_col_text(sqlite3_stmt * stmt,int col)658 get_stmt_col_text(sqlite3_stmt *stmt, int col)
659 {
660 const char *t = (const char *) sqlite3_column_text(stmt, col);
661 return t == NULL ? "*?*" : t;
662 }
663
664 /*
665 * Execute the full text search query and return the number of results
666 * obtained.
667 */
668 static int
execute_search_query(sqlite3 * db,char * query,query_args * args)669 execute_search_query(sqlite3 *db, char *query, query_args *args)
670 {
671 sqlite3_stmt *stmt;
672 char *name;
673 char *slash_ptr;
674 const char *name_temp;
675 char *m = NULL;
676 int rc;
677 query_callback_args callback_args;
678 inverse_document_frequency idf = {0, 0};
679
680 if (!args->legacy) {
681 /* Register the rank function */
682 rc = sqlite3_create_function(db, "rank_func", 1, SQLITE_ANY,
683 (void *) &idf, rank_func, NULL, NULL);
684 if (rc != SQLITE_OK) {
685 warnx("Unable to register the ranking function: %s",
686 sqlite3_errmsg(db));
687 sqlite3_close(db);
688 sqlite3_shutdown();
689 exit(EXIT_FAILURE);
690 }
691 }
692
693 rc = sqlite3_prepare_v2(db, query, -1, &stmt, NULL);
694 if (rc == SQLITE_IOERR) {
695 warnx("Corrupt database. Please rerun makemandb");
696 return -1;
697 } else if (rc != SQLITE_OK) {
698 warnx("%s", sqlite3_errmsg(db));
699 return -1;
700 }
701
702 int nresults = rc = 0;
703 while (rc == 0 && sqlite3_step(stmt) == SQLITE_ROW) {
704 nresults++;
705 callback_args.section = get_stmt_col_text(stmt, 0);
706 name_temp = get_stmt_col_text(stmt, 1);
707 callback_args.name_desc = get_stmt_col_text(stmt, 2);
708 callback_args.machine = (const char *) sqlite3_column_text(stmt, 3);
709 if (!args->legacy) {
710 callback_args.snippet = get_stmt_col_text(stmt, 4);
711 callback_args.snippet_length =
712 strlen(callback_args.snippet);
713 } else {
714 callback_args.snippet = "";
715 callback_args.snippet_length = 1;
716 }
717 if ((slash_ptr = strrchr(name_temp, '/')) != NULL)
718 name_temp = slash_ptr + 1;
719 if (callback_args.machine && callback_args.machine[0]) {
720 m = estrdup(callback_args.machine);
721 easprintf(&name, "%s/%s", lower(m), name_temp);
722 free(m);
723 } else {
724 name = estrdup(get_stmt_col_text(stmt, 1));
725 }
726 callback_args.name = name;
727 callback_args.other_data = args->callback_data;
728 rc = (args->callback)(&callback_args);
729 free(name);
730 }
731 sqlite3_finalize(stmt);
732 return (rc < 0) ? rc : nresults;
733 }
734
735
736 /*
737 * run_query_internal --
738 * Performs the searches for the keywords entered by the user.
739 * The 2nd param: snippet_args is an array of strings providing values for the
740 * last three parameters to the snippet function of sqlite. (Look at the docs).
741 * The 3rd param: args contains rest of the search parameters. Look at
742 * arpopos-utils.h for the description of individual fields.
743 *
744 */
745 static int
run_query_internal(sqlite3 * db,const char * snippet_args[3],query_args * args)746 run_query_internal(sqlite3 *db, const char *snippet_args[3], query_args *args)
747 {
748 char *query;
749 query = generate_search_query(args, snippet_args);
750 if (query == NULL) {
751 *args->errmsg = estrdup("malloc failed");
752 return -1;
753 }
754
755 int rc = execute_search_query(db, query, args);
756 sqlite3_free(query);
757 return (rc < 0 || *(args->errmsg) != NULL) ? -1 : 0;
758 }
759
760 static char *
get_escaped_html_string(const char * src,size_t * slen)761 get_escaped_html_string(const char *src, size_t *slen)
762 {
763 static const char trouble[] = "<>\"&\002\003";
764 /*
765 * First scan the src to find out the number of occurrences
766 * of {'>', '<' '"', '&'}. Then allocate a new buffer with
767 * sufficient space to be able to store the quoted versions
768 * of the special characters {>, <, ", &}.
769 * Copy over the characters from the original src into
770 * this buffer while replacing the special characters with
771 * their quoted versions.
772 */
773 char *dst, *ddst;
774 size_t count;
775 const char *ssrc;
776
777 for (count = 0, ssrc = src; *src; count++) {
778 size_t sz = strcspn(src, trouble);
779 src += sz + 1;
780 }
781
782
783 #define append(a) \
784 do { \
785 memcpy(dst, (a), sizeof(a) - 1); \
786 dst += sizeof(a) - 1; \
787 } while (0)
788
789
790 ddst = dst = emalloc(*slen + count * 5 + 1);
791 for (src = ssrc; *src; src++) {
792 switch (*src) {
793 case '<':
794 append("<");
795 break;
796 case '>':
797 append(">");
798 break;
799 case '\"':
800 append(""");
801 break;
802 case '&':
803 /*
804 * Don't perform the quoting if this & is part of
805 * an mdoc escape sequence, e.g. \&
806 */
807 if (src != ssrc && src[-1] != '\\')
808 append("&");
809 else
810 append("&");
811 break;
812 case '\002':
813 append("<b>");
814 break;
815 case '\003':
816 append("</b>");
817 break;
818 default:
819 *dst++ = *src;
820 break;
821 }
822 }
823 *dst = '\0';
824 *slen = dst - ddst;
825 return ddst;
826 }
827
828
829 /*
830 * callback_html --
831 * Callback function for run_query_html. It builds the html output and then
832 * calls the actual user supplied callback function.
833 */
834 static int
callback_html(query_callback_args * callback_args)835 callback_html(query_callback_args *callback_args)
836 {
837 struct orig_callback_data *orig_data = callback_args->other_data;
838 int (*callback)(query_callback_args*) = orig_data->callback;
839 size_t length = callback_args->snippet_length;
840 size_t name_description_length = strlen(callback_args->name_desc);
841 char *qsnippet = get_escaped_html_string(callback_args->snippet, &length);
842 char *qname_description = get_escaped_html_string(callback_args->name_desc,
843 &name_description_length);
844 callback_args->name_desc = qname_description;
845 callback_args->snippet = qsnippet;
846 callback_args->snippet_length = length;
847 callback_args->other_data = orig_data->data;
848 int rc = (*callback)(callback_args);
849 free(qsnippet);
850 free(qname_description);
851 return rc;
852 }
853
854 /*
855 * run_query_html --
856 * Utility function to output query result in HTML format.
857 * It internally calls run_query only, but it first passes the output to its
858 * own custom callback function, which preprocess the snippet for quoting
859 * inline HTML fragments.
860 * After that it delegates the call the actual user supplied callback function.
861 */
862 static int
run_query_html(sqlite3 * db,query_args * args)863 run_query_html(sqlite3 *db, query_args *args)
864 {
865 struct orig_callback_data orig_data;
866 orig_data.callback = args->callback;
867 orig_data.data = args->callback_data;
868 const char *snippet_args[] = {"\002", "\003", "..."};
869 args->callback = &callback_html;
870 args->callback_data = (void *) &orig_data;
871 return run_query_internal(db, snippet_args, args);
872 }
873
874 /*
875 * underline a string, pager style.
876 */
877 static char *
ul_pager(int ul,const char * s)878 ul_pager(int ul, const char *s)
879 {
880 size_t len;
881 char *dst, *d;
882
883 if (!ul)
884 return estrdup(s);
885
886 // a -> _\ba
887 len = strlen(s) * 3 + 1;
888
889 d = dst = emalloc(len);
890 while (*s) {
891 *d++ = '_';
892 *d++ = '\b';
893 *d++ = *s++;
894 }
895 *d = '\0';
896 return dst;
897 }
898
899 /*
900 * callback_pager --
901 * A callback similar to callback_html. It overstrikes the matching text in
902 * the snippet so that it appears emboldened when viewed using a pager like
903 * more or less.
904 */
905 static int
callback_pager(query_callback_args * callback_args)906 callback_pager(query_callback_args *callback_args)
907 {
908 struct orig_callback_data *orig_data = callback_args->other_data;
909 char *psnippet;
910 const char *temp = callback_args->snippet;
911 int count = 0;
912 int i = 0, did;
913 size_t sz = 0;
914 size_t psnippet_length;
915
916 /* Count the number of bytes of matching text. For each of these
917 * bytes we will use 2 extra bytes to overstrike it so that it
918 * appears bold when viewed using a pager.
919 */
920 while (*temp) {
921 sz = strcspn(temp, "\002\003");
922 temp += sz;
923 if (*temp == '\003') {
924 count += 2 * (sz);
925 }
926 temp++;
927 }
928
929 psnippet_length = callback_args->snippet_length + count;
930 psnippet = emalloc(psnippet_length + 1);
931
932 /* Copy the bytes from snippet to psnippet:
933 * 1. Copy the bytes before \002 as it is.
934 * 2. The bytes after \002 need to be overstriked till we
935 * encounter \003.
936 * 3. To overstrike a byte 'A' we need to write 'A\bA'
937 */
938 did = 0;
939 const char *snippet = callback_args->snippet;
940 while (*snippet) {
941 sz = strcspn(snippet, "\002");
942 memcpy(&psnippet[i], snippet, sz);
943 snippet += sz;
944 i += sz;
945
946 /* Don't change this. Advancing the pointer without reading the byte
947 * is causing strange behavior.
948 */
949 if (*snippet == '\002')
950 snippet++;
951 while (*snippet && *snippet != '\003') {
952 did = 1;
953 psnippet[i++] = *snippet;
954 psnippet[i++] = '\b';
955 psnippet[i++] = *snippet++;
956 }
957 if (*snippet)
958 snippet++;
959 }
960
961 psnippet[i] = 0;
962 char *ul_section = ul_pager(did, callback_args->section);
963 char *ul_name = ul_pager(did, callback_args->name);
964 char *ul_name_desc = ul_pager(did, callback_args->name_desc);
965 callback_args->section = ul_section;
966 callback_args->name = ul_name;
967 callback_args->name_desc = ul_name_desc;
968 callback_args->snippet = psnippet;
969 callback_args->snippet_length = psnippet_length;
970 callback_args->other_data = orig_data->data;
971 int rc = (orig_data->callback)(callback_args);
972 free(ul_section);
973 free(ul_name);
974 free(ul_name_desc);
975 free(psnippet);
976 return rc;
977 }
978
979 struct term_args {
980 struct orig_callback_data *orig_data;
981 const char *smul;
982 const char *rmul;
983 };
984
985 /*
986 * underline a string, pager style.
987 */
988 static char *
ul_term(const char * s,const struct term_args * ta)989 ul_term(const char *s, const struct term_args *ta)
990 {
991 char *dst;
992
993 easprintf(&dst, "%s%s%s", ta->smul, s, ta->rmul);
994 return dst;
995 }
996
997 /*
998 * callback_term --
999 * A callback similar to callback_html. It overstrikes the matching text in
1000 * the snippet so that it appears emboldened when viewed using a pager like
1001 * more or less.
1002 */
1003 static int
callback_term(query_callback_args * callback_args)1004 callback_term(query_callback_args *callback_args)
1005 {
1006 struct term_args *ta = callback_args->other_data;
1007 struct orig_callback_data *orig_data = ta->orig_data;
1008
1009 char *ul_section = ul_term(callback_args->section, ta);
1010 char *ul_name = ul_term(callback_args->name, ta);
1011 char *ul_name_desc = ul_term(callback_args->name_desc, ta);
1012 callback_args->section = ul_section;
1013 callback_args->name = ul_name;
1014 callback_args->name_desc = ul_name_desc;
1015 callback_args->other_data = orig_data->data;
1016 int rc = (orig_data->callback)(callback_args);
1017 free(ul_section);
1018 free(ul_name);
1019 free(ul_name_desc);
1020 return rc;
1021 }
1022
1023 /*
1024 * run_query_pager --
1025 * Utility function similar to run_query_html. This function tries to
1026 * pre-process the result assuming it will be piped to a pager.
1027 * For this purpose it first calls its own callback function callback_pager
1028 * which then delegates the call to the user supplied callback.
1029 */
1030 static int
run_query_pager(sqlite3 * db,query_args * args)1031 run_query_pager(sqlite3 *db, query_args *args)
1032 {
1033 struct orig_callback_data orig_data;
1034 orig_data.callback = args->callback;
1035 orig_data.data = args->callback_data;
1036 const char *snippet_args[3] = { "\002", "\003", "..." };
1037 args->callback = &callback_pager;
1038 args->callback_data = (void *) &orig_data;
1039 return run_query_internal(db, snippet_args, args);
1040 }
1041
1042 struct nv {
1043 char *s;
1044 size_t l;
1045 };
1046
1047 static int
term_putc(int c,void * p)1048 term_putc(int c, void *p)
1049 {
1050 struct nv *nv = p;
1051 nv->s[nv->l++] = c;
1052 return 0;
1053 }
1054
1055 static char *
term_fix_seq(TERMINAL * ti,const char * seq)1056 term_fix_seq(TERMINAL *ti, const char *seq)
1057 {
1058 char *res = estrdup(seq);
1059 struct nv nv;
1060
1061 if (ti == NULL)
1062 return res;
1063
1064 nv.s = res;
1065 nv.l = 0;
1066 ti_puts(ti, seq, 1, term_putc, &nv);
1067 nv.s[nv.l] = '\0';
1068
1069 return res;
1070 }
1071
1072 static void
term_init(int fd,const char * sa[5])1073 term_init(int fd, const char *sa[5])
1074 {
1075 TERMINAL *ti;
1076 int error;
1077 const char *bold, *sgr0, *smso, *rmso, *smul, *rmul;
1078
1079 if (ti_setupterm(&ti, NULL, fd, &error) == -1) {
1080 bold = sgr0 = NULL;
1081 smso = rmso = smul = rmul = "";
1082 ti = NULL;
1083 } else {
1084 bold = ti_getstr(ti, "bold");
1085 sgr0 = ti_getstr(ti, "sgr0");
1086 if (bold == NULL || sgr0 == NULL) {
1087 smso = ti_getstr(ti, "smso");
1088
1089 if (smso == NULL ||
1090 (rmso = ti_getstr(ti, "rmso")) == NULL)
1091 smso = rmso = "";
1092 bold = sgr0 = NULL;
1093 } else
1094 smso = rmso = "";
1095
1096 smul = ti_getstr(ti, "smul");
1097 if (smul == NULL || (rmul = ti_getstr(ti, "rmul")) == NULL)
1098 smul = rmul = "";
1099 }
1100
1101 sa[0] = term_fix_seq(ti, bold ? bold : smso);
1102 sa[1] = term_fix_seq(ti, sgr0 ? sgr0 : rmso);
1103 sa[2] = estrdup("...");
1104 sa[3] = term_fix_seq(ti, smul);
1105 sa[4] = term_fix_seq(ti, rmul);
1106
1107 if (ti)
1108 del_curterm(ti);
1109 }
1110
1111 /*
1112 * run_query_term --
1113 * Utility function similar to run_query_html. This function tries to
1114 * pre-process the result assuming it will be displayed on a terminal
1115 * For this purpose it first calls its own callback function callback_pager
1116 * which then delegates the call to the user supplied callback.
1117 */
1118 static int
run_query_term(sqlite3 * db,query_args * args)1119 run_query_term(sqlite3 *db, query_args *args)
1120 {
1121 struct orig_callback_data orig_data;
1122 struct term_args ta;
1123 orig_data.callback = args->callback;
1124 orig_data.data = args->callback_data;
1125 const char *snippet_args[5];
1126
1127 term_init(STDOUT_FILENO, snippet_args);
1128 ta.smul = snippet_args[3];
1129 ta.rmul = snippet_args[4];
1130 ta.orig_data = (void *) &orig_data;
1131
1132 args->callback = &callback_term;
1133 args->callback_data = &ta;
1134 return run_query_internal(db, snippet_args, args);
1135 }
1136
1137 static int
run_query_none(sqlite3 * db,query_args * args)1138 run_query_none(sqlite3 *db, query_args *args)
1139 {
1140 struct orig_callback_data orig_data;
1141 orig_data.callback = args->callback;
1142 orig_data.data = args->callback_data;
1143 const char *snippet_args[3] = { "", "", "..." };
1144 args->callback = &callback_pager;
1145 args->callback_data = (void *) &orig_data;
1146 return run_query_internal(db, snippet_args, args);
1147 }
1148
1149 int
run_query(sqlite3 * db,query_format fmt,query_args * args)1150 run_query(sqlite3 *db, query_format fmt, query_args *args)
1151 {
1152 switch (fmt) {
1153 case APROPOS_NONE:
1154 return run_query_none(db, args);
1155 case APROPOS_HTML:
1156 return run_query_html(db, args);
1157 case APROPOS_TERM:
1158 return run_query_term(db, args);
1159 case APROPOS_PAGER:
1160 return run_query_pager(db, args);
1161 default:
1162 warnx("Unknown query format %d", (int)fmt);
1163 return -1;
1164 }
1165 }
1166