Fossil

Check-in [0e77f1fb]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Show document, ticket, and wiki titles on the result page of unindexed search.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | search-enhancements
Files: files | file ages | folders
SHA1:0e77f1fbc08a5f1f8acf758a7b882e85c4cde584
User & Date: drh 2015-02-13 23:43:59
Context
2015-02-14
00:37
Enabled indexed search with separate title and body and with the option to use the Porter stemmer. check-in: 71295a98 user: drh tags: search-enhancements
2015-02-13
23:43
Show document, ticket, and wiki titles on the result page of unindexed search. check-in: 0e77f1fb user: drh tags: search-enhancements
22:03
When extracting search text from tickets, use the title column for the title and pay attention to mimetypes when translating. check-in: b552f55b user: drh tags: search-enhancements
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to src/search.c.

211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
...
394
395
396
397
398
399
400
401

402
403




404
405
406
407
408
409
410
411
...
433
434
435
436
437
438
439
440










441
442
443

444

445















446
447
448
449
450
451
452
...
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477




478
479
480
481
482
483
484
...
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630

631
632
633
634
635
636
637
638
...
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
...
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
....
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
....
1168
1169
1170
1171
1172
1173
1174









































1175
1176
1177
1178
1179
1180
1181
        ){
          aiWordIdx[j] = iWord;
          aiLastDoc[j] = iDoc;
          aiLastOfst[j] = i;
          for(k=1; j-k>=0 && anMatch[j-k] && aiWordIdx[j-k]==iWord-k; k++){}
          for(ii=0; ii<k; ii++){
            if( anMatch[j-ii]<k ){
              anMatch[j-ii] = k;
              aiBestDoc[j-ii] = aiLastDoc[j-ii];
              aiBestOfst[j-ii] = aiLastOfst[j-ii];
            }
          }
          break;
        }
      }
................................................................................
** Return non-zero on a match and zero on a miss.
*/
static void search_match_sqlfunc(
  sqlite3_context *context,
  int argc,
  sqlite3_value **argv
){
  const char *zSText = (const char*)sqlite3_value_text(argv[0]);

  int rc;
  if( zSText==0 ) return;




  rc = search_match(&gSearch, 1, &zSText);
  sqlite3_result_int(context, rc);
}

/*
** These SQL functions return the results of the last
** call to the search_match() SQL function.
*/
................................................................................
** search_stext() routine for further detail.
*/
static void search_stext_sqlfunc(
  sqlite3_context *context,
  int argc,
  sqlite3_value **argv
){
  Blob txt;










  const char *zType = (const char*)sqlite3_value_text(argv[0]);
  int rid = sqlite3_value_int(argv[1]);
  const char *zName = (const char*)sqlite3_value_text(argv[2]);

  search_stext(zType[0], rid, zName, &txt);

  sqlite3_result_text(context, blob_materialize(&txt), -1, fossil_free);















}

/*
** Encode a string for use as a query parameter in a URL
*/
static void search_urlencode_sqlfunc(
  sqlite3_context *context,
................................................................................
** Register the "score()" SQL function to score its input text
** using the given Search object.  Once this function is registered,
** do not delete the Search object.
*/
void search_sql_setup(sqlite3 *db){
  static int once = 0;
  if( once++ ) return;
  sqlite3_create_function(db, "search_match", 1, SQLITE_UTF8, 0,
     search_match_sqlfunc, 0, 0);
  sqlite3_create_function(db, "search_score", 0, SQLITE_UTF8, 0,
     search_score_sqlfunc, 0, 0);
  sqlite3_create_function(db, "search_snippet", 0, SQLITE_UTF8, 0,
     search_snippet_sqlfunc, 0, 0);
  sqlite3_create_function(db, "search_init", -1, SQLITE_UTF8, 0,
     search_init_sqlfunc, 0, 0);
  sqlite3_create_function(db, "stext", 3, SQLITE_UTF8, 0,
     search_stext_sqlfunc, 0, 0);




  sqlite3_create_function(db, "urlencode", 1, SQLITE_UTF8, 0,
     search_urlencode_sqlfunc, 0, 0);
}

/*
** Testing the search function.
**
................................................................................
    char *zDocBr = db_get("doc-branch","trunk");
    if( zDocGlob && zDocGlob[0] && zDocBr && zDocBr[0] ){
      db_multi_exec(
        "CREATE VIRTUAL TABLE IF NOT EXISTS temp.foci USING files_of_checkin;"
      );
      db_multi_exec(
        "INSERT INTO x(label,url,score,date,snip)"
        "  SELECT printf('Document: %%s',foci.filename),"
        "         printf('/doc/%T/%%s',foci.filename),"
        "         search_score(),"
        "         (SELECT datetime(event.mtime) FROM event"
        "            WHERE objid=symbolic_name_to_rid('trunk')),"
        "         search_snippet()"
        "    FROM foci CROSS JOIN blob"
        "   WHERE checkinID=symbolic_name_to_rid('trunk')"
        "     AND blob.uuid=foci.uuid"

        "     AND search_match(stext('d',blob.rid,foci.filename))"
        "     AND %z",
        zDocBr, glob_expr("foci.filename", zDocGlob)
      );
    }
  }
  if( (srchFlags & SRCH_WIKI)!=0 ){
    db_multi_exec(
................................................................................
      "INSERT INTO x(label,url,score,date,snip)"
      "  SELECT printf('Wiki: %%s',name),"
      "         printf('/wiki?name=%%s',urlencode(name)),"
      "         search_score(),"
      "         datetime(mtime),"
      "         search_snippet()"
      "    FROM wiki"
      "   WHERE search_match(stext('w',rid,name));"
    );
  }
  if( (srchFlags & SRCH_CKIN)!=0 ){
    db_multi_exec(
      "WITH ckin(uuid,rid,mtime) AS ("
      "  SELECT blob.uuid, event.objid, event.mtime"
      "    FROM event, blob"
................................................................................
      "INSERT INTO x(label,url,score,date,snip)"
      "  SELECT printf('Check-in [%%.10s] on %%s',uuid,datetime(mtime)),"
      "         printf('/timeline?c=%%s&n=8&y=ci',uuid),"
      "         search_score(),"
      "         datetime(mtime),"
      "         search_snippet()"
      "    FROM ckin"
      "   WHERE search_match(stext('c',rid,NULL));"
    );
  }
  if( (srchFlags & SRCH_TKT)!=0 ){
    db_multi_exec(
      "INSERT INTO x(label,url,score, date,snip)"
      "  SELECT printf('Ticket [%%.17s] on %%s',"
                      "tkt_uuid,datetime(tkt_mtime)),"
      "         printf('/tktview/%%.20s',tkt_uuid),"
      "         search_score(),"
      "         datetime(tkt_mtime),"
      "         search_snippet()"
      "    FROM ticket"
      "   WHERE search_match(stext('t',tkt_id,NULL));"
    );
  }
}

/*
** Implemenation of the rank() function used with rank(matchinfo(*,'pcsx')).
*/
................................................................................
    html_to_plaintext(blob_str(&html), pOut);
  }else if( fossil_strcmp(zMimetype,"text/html")==0 ){
    if( doc_is_embedded_html(pIn, &title) ){
      blob_appendf(pOut, "%s\n", blob_str(&title));
    }
    html_to_plaintext(blob_str(pIn), pOut);
  }else{
    *pOut = *pIn;
    blob_init(pIn, 0, 0);
  }
  blob_reset(&html);
  blob_reset(&title);
}

/*
** Query pQuery is pointing at a single row of output.  Append a text
** representation of every text-compatible column to pAccum.
*/
static void append_all_ticket_fields(Blob *pAccum, Stmt *pQuery, int iTitle){
  int n = db_column_count(pQuery);
  int i;
  const char *zMime = 0;
  if( iTitle>=0 ){
    if( db_column_type(pQuery,iTitle)==SQLITE_TEXT ){
      blob_append(pAccum, db_column_text(pQuery,iTitle), -1);
    }
    blob_append(pAccum, "\n", 1);
  }
  for(i=0; i<n; i++){
    const char *zColName = db_column_name(pQuery,i);
................................................................................
        }
        db_reset(&q2);
      }
      break;
    }
  }
}










































/*
** COMMAND: test-search-stext
**
** Usage: fossil test-search-stext TYPE ARG1 ARG2
*/
void test_search_stext(void){







|







 







|
>

<
>
>
>
>
|







 







|
>
>
>
>
>
>
>
>
>
>



>
|
>
|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







 







|









>
>
>
>







 







|








>
|







 







|







 







|





|
|





|







 







|
<













|







 







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
...
394
395
396
397
398
399
400
401
402
403

404
405
406
407
408
409
410
411
412
413
414
415
...
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
...
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
...
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
...
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
...
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
....
1057
1058
1059
1060
1061
1062
1063
1064

1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
....
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
        ){
          aiWordIdx[j] = iWord;
          aiLastDoc[j] = iDoc;
          aiLastOfst[j] = i;
          for(k=1; j-k>=0 && anMatch[j-k] && aiWordIdx[j-k]==iWord-k; k++){}
          for(ii=0; ii<k; ii++){
            if( anMatch[j-ii]<k ){
              anMatch[j-ii] = k*(nDoc-iDoc);
              aiBestDoc[j-ii] = aiLastDoc[j-ii];
              aiBestOfst[j-ii] = aiLastOfst[j-ii];
            }
          }
          break;
        }
      }
................................................................................
** Return non-zero on a match and zero on a miss.
*/
static void search_match_sqlfunc(
  sqlite3_context *context,
  int argc,
  sqlite3_value **argv
){
  const char *azDoc[5];
  int nDoc;
  int rc;

  for(nDoc=0; nDoc<ArraySize(azDoc) && nDoc<argc; nDoc++){
    azDoc[nDoc] = (const char*)sqlite3_value_text(argv[nDoc]);
    if( azDoc[nDoc]==0 ) azDoc[nDoc] = "";
  }
  rc = search_match(&gSearch, nDoc, azDoc);
  sqlite3_result_int(context, rc);
}

/*
** These SQL functions return the results of the last
** call to the search_match() SQL function.
*/
................................................................................
** search_stext() routine for further detail.
*/
static void search_stext_sqlfunc(
  sqlite3_context *context,
  int argc,
  sqlite3_value **argv
){
  const char *zType = (const char*)sqlite3_value_text(argv[0]);
  int rid = sqlite3_value_int(argv[1]);
  const char *zName = (const char*)sqlite3_value_text(argv[2]);
  sqlite3_result_text(context, search_stext_cached(zType[0],rid,zName,0), -1,
                      SQLITE_TRANSIENT);
}
static void search_title_sqlfunc(
  sqlite3_context *context,
  int argc,
  sqlite3_value **argv
){
  const char *zType = (const char*)sqlite3_value_text(argv[0]);
  int rid = sqlite3_value_int(argv[1]);
  const char *zName = (const char*)sqlite3_value_text(argv[2]);
  int nHdr;
  char *z = search_stext_cached(zType[0], rid, zName, &nHdr);
  if( nHdr || zType[0]!='d' ){
    sqlite3_result_text(context, z, nHdr, SQLITE_TRANSIENT);
  }else{
    sqlite3_result_value(context, argv[2]);
  }
}
static void search_body_sqlfunc(
  sqlite3_context *context,
  int argc,
  sqlite3_value **argv
){
  const char *zType = (const char*)sqlite3_value_text(argv[0]);
  int rid = sqlite3_value_int(argv[1]);
  const char *zName = (const char*)sqlite3_value_text(argv[2]);
  int nHdr;
  char *z = search_stext_cached(zType[0], rid, zName, &nHdr);
  sqlite3_result_text(context, z+nHdr+1, -1, SQLITE_TRANSIENT);
}

/*
** Encode a string for use as a query parameter in a URL
*/
static void search_urlencode_sqlfunc(
  sqlite3_context *context,
................................................................................
** Register the "score()" SQL function to score its input text
** using the given Search object.  Once this function is registered,
** do not delete the Search object.
*/
void search_sql_setup(sqlite3 *db){
  static int once = 0;
  if( once++ ) return;
  sqlite3_create_function(db, "search_match", -1, SQLITE_UTF8, 0,
     search_match_sqlfunc, 0, 0);
  sqlite3_create_function(db, "search_score", 0, SQLITE_UTF8, 0,
     search_score_sqlfunc, 0, 0);
  sqlite3_create_function(db, "search_snippet", 0, SQLITE_UTF8, 0,
     search_snippet_sqlfunc, 0, 0);
  sqlite3_create_function(db, "search_init", -1, SQLITE_UTF8, 0,
     search_init_sqlfunc, 0, 0);
  sqlite3_create_function(db, "stext", 3, SQLITE_UTF8, 0,
     search_stext_sqlfunc, 0, 0);
  sqlite3_create_function(db, "title", 3, SQLITE_UTF8, 0,
     search_title_sqlfunc, 0, 0);
  sqlite3_create_function(db, "body", 3, SQLITE_UTF8, 0,
     search_body_sqlfunc, 0, 0);
  sqlite3_create_function(db, "urlencode", 1, SQLITE_UTF8, 0,
     search_urlencode_sqlfunc, 0, 0);
}

/*
** Testing the search function.
**
................................................................................
    char *zDocBr = db_get("doc-branch","trunk");
    if( zDocGlob && zDocGlob[0] && zDocBr && zDocBr[0] ){
      db_multi_exec(
        "CREATE VIRTUAL TABLE IF NOT EXISTS temp.foci USING files_of_checkin;"
      );
      db_multi_exec(
        "INSERT INTO x(label,url,score,date,snip)"
        "  SELECT printf('Document: %%s',title('d',blob.rid,foci.filename)),"
        "         printf('/doc/%T/%%s',foci.filename),"
        "         search_score(),"
        "         (SELECT datetime(event.mtime) FROM event"
        "            WHERE objid=symbolic_name_to_rid('trunk')),"
        "         search_snippet()"
        "    FROM foci CROSS JOIN blob"
        "   WHERE checkinID=symbolic_name_to_rid('trunk')"
        "     AND blob.uuid=foci.uuid"
        "     AND search_match(title('d',blob.rid,foci.filename),"
        "                      body('d',blob.rid,foci.filename))"
        "     AND %z",
        zDocBr, glob_expr("foci.filename", zDocGlob)
      );
    }
  }
  if( (srchFlags & SRCH_WIKI)!=0 ){
    db_multi_exec(
................................................................................
      "INSERT INTO x(label,url,score,date,snip)"
      "  SELECT printf('Wiki: %%s',name),"
      "         printf('/wiki?name=%%s',urlencode(name)),"
      "         search_score(),"
      "         datetime(mtime),"
      "         search_snippet()"
      "    FROM wiki"
      "   WHERE search_match(title('w',rid,name),body('w',rid,name));"
    );
  }
  if( (srchFlags & SRCH_CKIN)!=0 ){
    db_multi_exec(
      "WITH ckin(uuid,rid,mtime) AS ("
      "  SELECT blob.uuid, event.objid, event.mtime"
      "    FROM event, blob"
................................................................................
      "INSERT INTO x(label,url,score,date,snip)"
      "  SELECT printf('Check-in [%%.10s] on %%s',uuid,datetime(mtime)),"
      "         printf('/timeline?c=%%s&n=8&y=ci',uuid),"
      "         search_score(),"
      "         datetime(mtime),"
      "         search_snippet()"
      "    FROM ckin"
      "   WHERE search_match('',body('c',rid,NULL));"
    );
  }
  if( (srchFlags & SRCH_TKT)!=0 ){
    db_multi_exec(
      "INSERT INTO x(label,url,score, date,snip)"
      "  SELECT printf('Ticket: %%s (%%s)',title('t',tkt_id,NULL),"
                      "datetime(tkt_mtime)),"
      "         printf('/tktview/%%.20s',tkt_uuid),"
      "         search_score(),"
      "         datetime(tkt_mtime),"
      "         search_snippet()"
      "    FROM ticket"
      "   WHERE search_match(title('t',tkt_id,NULL),body('t',tkt_id,NULL));"
    );
  }
}

/*
** Implemenation of the rank() function used with rank(matchinfo(*,'pcsx')).
*/
................................................................................
    html_to_plaintext(blob_str(&html), pOut);
  }else if( fossil_strcmp(zMimetype,"text/html")==0 ){
    if( doc_is_embedded_html(pIn, &title) ){
      blob_appendf(pOut, "%s\n", blob_str(&title));
    }
    html_to_plaintext(blob_str(pIn), pOut);
  }else{
    blob_append(pOut, blob_buffer(pIn), blob_size(pIn));

  }
  blob_reset(&html);
  blob_reset(&title);
}

/*
** Query pQuery is pointing at a single row of output.  Append a text
** representation of every text-compatible column to pAccum.
*/
static void append_all_ticket_fields(Blob *pAccum, Stmt *pQuery, int iTitle){
  int n = db_column_count(pQuery);
  int i;
  const char *zMime = 0;
  if( iTitle>=0 && iTitle<n ){
    if( db_column_type(pQuery,iTitle)==SQLITE_TEXT ){
      blob_append(pAccum, db_column_text(pQuery,iTitle), -1);
    }
    blob_append(pAccum, "\n", 1);
  }
  for(i=0; i<n; i++){
    const char *zColName = db_column_name(pQuery,i);
................................................................................
        }
        db_reset(&q2);
      }
      break;
    }
  }
}

/*
** This routine is a wrapper around search_stext().  
**
** This routine looks up the search text, stores it in an internal
** buffer, and returns a pointer to the text.  Subsequent requests
** for the same document return the same pointer.  The returned pointer
** is valid until the next invocation of this routine.  Call this routine
** with an eType of 0 to clear the cache.
*/
char *search_stext_cached(
  char cType,            /* Type of document */
  int rid,               /* BLOB.RID or TAG.TAGID value for document */
  const char *zName,     /* Auxiliary information */
  int *pnTitle           /* OUT: length of title in bytes excluding \n */
){
  static struct {
    Blob stext;          /* Cached search text */
    char cType;          /* The type */
    int rid;             /* The RID */
    int nTitle;          /* Number of bytes in title */
  } cache;
  int i;
  char *z;
  if( cType!=cache.cType || rid!=cache.rid ){
    if( cache.rid>0 ){
      blob_reset(&cache.stext);
    }else{
      blob_init(&cache.stext,0,0);
    }
    cache.cType = cType;
    cache.rid = rid;
    if( cType==0 ) return 0;
    search_stext(cType, rid, zName, &cache.stext);
    z  = blob_str(&cache.stext);
    for(i=0; z[i] && z[i]!='\n'; i++){}
    cache.nTitle = i;
  }
  if( pnTitle ) *pnTitle = cache.nTitle;
  return blob_str(&cache.stext);
}

/*
** COMMAND: test-search-stext
**
** Usage: fossil test-search-stext TYPE ARG1 ARG2
*/
void test_search_stext(void){