Skip to content

Commit 4150991

Browse files
MetaMemoryTMetaMemoryT
committed
added a character tokenizer for fts, TODO seperate this functionality into its own package
1 parent f69b89a commit 4150991

7 files changed

Lines changed: 478 additions & 1 deletion

File tree

‎binding.gyp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,8 @@
2727
"sources": [
2828
"src/database.cc",
2929
"src/node_sqlite3.cc",
30-
"src/statement.cc"
30+
"src/statement.cc",
31+
"src/character_tokenizer.cc"
3132
]
3233
},
3334
{

‎src/character_tokenizer.cc‎

Lines changed: 170 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,170 @@
1+
//
2+
// character_tokenizer.c
3+
//
4+
// Created by Hai Feng Kao on 4/6/13.
5+
// All rights reserved.
6+
//
7+
// Permission is hereby granted, free of charge, to any person obtaining a copy
8+
// of this software and associated documentation files (the "Software"), to deal
9+
// in the Software without restriction, including without limitation the rights
10+
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
11+
// copies of the Software, and to permit persons to whom the Software is
12+
// furnished to do so, subject to the following conditions:
13+
//
14+
// The above copyright notice and this permission notice shall be included in
15+
// all copies or substantial portions of the Software.
16+
//
17+
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18+
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19+
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
20+
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21+
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
22+
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
23+
// THE SOFTWARE.
24+
//
25+
// Implementation of the "simple" full-text-search tokenizer.
26+
27+
#include <sqlite3.h>
28+
#include <ctype.h> //for tolower
29+
#include <string.h> //for memset
30+
#include "character_tokenizer.h"
31+
32+
typedef struct character_tokenizer {
33+
sqlite3_tokenizer base;
34+
} character_tokenizer;
35+
36+
typedef struct character_tokenizer_cursor {
37+
sqlite3_tokenizer_cursor base;
38+
const char *pInput; // input we are tokenizing
39+
int nBytes; // size of the input
40+
int iPosition; // current position in pInput
41+
int iToken; // index of next token to be returned
42+
char *pToken; // storage for current token
43+
} character_tokenizer_cursor;
44+
45+
static int characterCreate(
46+
int argc, const char * const *argv,
47+
sqlite3_tokenizer **ppTokenizer
48+
){
49+
character_tokenizer *t;
50+
t = (character_tokenizer *) sqlite3_malloc(sizeof(*t));
51+
if( t == NULL
52+
) return SQLITE_NOMEM;
53+
memset(t, 0, sizeof(*t));
54+
55+
*ppTokenizer = &t->base;
56+
return SQLITE_OK;
57+
}
58+
59+
static int characterDestroy(sqlite3_tokenizer *pTokenizer){
60+
sqlite3_free(pTokenizer);
61+
return SQLITE_OK;
62+
}
63+
64+
static int characterOpen(
65+
sqlite3_tokenizer *pTokenizer, /* The tokenizer */
66+
const char *pInput, int nBytes, /* String to be tokenized */
67+
sqlite3_tokenizer_cursor **ppCursor /* OUT: Tokenization cursor */
68+
){
69+
character_tokenizer_cursor *c;
70+
if(pInput == 0){
71+
nBytes = 0;
72+
}else if(nBytes < 0){
73+
nBytes = (int)strlen(pInput);
74+
}
75+
c = (character_tokenizer_cursor *) sqlite3_malloc(sizeof(*c));
76+
if(c == NULL){
77+
return SQLITE_NOMEM;
78+
}
79+
c->iToken = c->iPosition = 0;
80+
c->pToken = NULL;
81+
c->nBytes = nBytes;
82+
c->pInput = pInput;
83+
*ppCursor = &c->base;
84+
return SQLITE_OK;
85+
}
86+
87+
static int characterClose(sqlite3_tokenizer_cursor *pCursor){
88+
character_tokenizer_cursor *c = (character_tokenizer_cursor *) pCursor;
89+
90+
if(c->pToken != NULL){
91+
sqlite3_free(c->pToken);
92+
c->pToken = NULL;
93+
}
94+
95+
sqlite3_free(c);
96+
return SQLITE_OK;
97+
}
98+
99+
static int characterNext(
100+
sqlite3_tokenizer_cursor *pCursor, /* Cursor returned by cusOpen */
101+
const char **ppToken, /* OUT: *ppToken is the token text */
102+
int *pnBytes, /* OUT: Number of bytes in token */
103+
int *piStartOffset, /* OUT: Starting offset of token */
104+
int *piEndOffset, /* OUT: Ending offset of token */
105+
int *piPosition /* OUT: Position integer of token */
106+
){
107+
character_tokenizer_cursor *c = (character_tokenizer_cursor *) pCursor;
108+
if(c->pToken != NULL){
109+
sqlite3_free(c->pToken);
110+
c->pToken = NULL;
111+
}
112+
113+
if (c->iPosition >= c->nBytes) {
114+
return SQLITE_DONE;
115+
}
116+
117+
int length = 1; // the size of current character, which can be at most 4 bytes
118+
119+
const char* token = &(c->pInput[c->iPosition]);
120+
*piStartOffset = c->iPosition;
121+
122+
// find the beginning of next utf8 character
123+
c->iPosition++;
124+
while (c->iPosition < c->nBytes) {
125+
char byte = c->pInput[c->iPosition];
126+
if (((byte & 0x80) == 0) || ((byte & 0xc0) == 0xc0)) {
127+
// we have reached the first byte of next utf8 character
128+
break;
129+
}
130+
length++;
131+
c->iPosition++;
132+
}
133+
134+
c->pToken = (char *)sqlite3_malloc(length+1);
135+
if(c->pToken == NULL){
136+
return SQLITE_NOMEM;
137+
}
138+
139+
c->pToken[length] = 0;
140+
memcpy(c->pToken, token, length);
141+
142+
for (int i = 0; i < length; ++i) {
143+
unsigned char byte = c->pToken[i];
144+
145+
if (byte < 0x80) {
146+
// ascii character, make it case-insensitive
147+
c->pToken[i] = tolower(byte);
148+
}
149+
}
150+
151+
*ppToken = c->pToken;
152+
*pnBytes = length;
153+
154+
*piEndOffset = *piStartOffset+length;
155+
*piPosition = c->iToken++;
156+
return SQLITE_OK;
157+
}
158+
159+
static const sqlite3_tokenizer_module characterTokenizerModule = {
160+
0,
161+
characterCreate,
162+
characterDestroy,
163+
characterOpen,
164+
characterClose,
165+
characterNext,
166+
};
167+
168+
void get_character_tokenizer_module(const sqlite3_tokenizer_module **ppModule){
169+
*ppModule = &characterTokenizerModule;
170+
}

‎src/character_tokenizer.h‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
//
2+
// character_tokenizer.h
3+
//
4+
// Created by Hai Feng Kao on 4/6/13.
5+
// All rights reserved.
6+
//
7+
// Permission is hereby granted, free of charge, to any person obtaining a copy
8+
// of this software and associated documentation files (the "Software"), to deal
9+
// in the Software without restriction, including without limitation the rights
10+
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
11+
// copies of the Software, and to permit persons to whom the Software is
12+
// furnished to do so, subject to the following conditions:
13+
//
14+
// The above copyright notice and this permission notice shall be included in
15+
// all copies or substantial portions of the Software.
16+
//
17+
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18+
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19+
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
20+
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21+
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
22+
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
23+
// THE SOFTWARE.
24+
//
25+
26+
#ifndef SQLITE_CHARACTERTOKENIZER_H
27+
#define SQLITE_CHARACTERTOKENIZER_H
28+
29+
#include "fts3_tokenizer.h"
30+
31+
void get_character_tokenizer_module(const sqlite3_tokenizer_module **ppModule);
32+
33+
#endif

‎src/database.cc‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
#include "macros.h"
55
#include "database.h"
66
#include "statement.h"
7+
#include "character_tokenizer.h"
78

89
using namespace node_sqlite3;
910

@@ -24,6 +25,8 @@ void Database::Init(Handle<Object> target) {
2425
NODE_SET_PROTOTYPE_METHOD(t, "serialize", Serialize);
2526
NODE_SET_PROTOTYPE_METHOD(t, "parallelize", Parallelize);
2627
NODE_SET_PROTOTYPE_METHOD(t, "configure", Configure);
28+
29+
NODE_SET_PROTOTYPE_METHOD(t, "loadCharacterTokenizer", LoadCharacterTokenizer);
2730

2831
NODE_SET_GETTER(t, "open", OpenGetter);
2932

@@ -589,6 +592,19 @@ void Database::Work_Wait(Baton* baton) {
589592
delete baton;
590593
}
591594

595+
NAN_METHOD(Database::LoadCharacterTokenizer) {
596+
NanScope();
597+
Database* db = ObjectWrap::Unwrap<Database>(args.This());
598+
599+
REQUIRE_ARGUMENT_STRING(0, filename);
600+
OPTIONAL_ARGUMENT_FUNCTION(1, callback);
601+
602+
Baton* baton = new LoadExtensionBaton(db, callback, *filename);
603+
db->Schedule(Work_BeginLoadCharacterTokenizer, baton, true);
604+
605+
NanReturnValue(args.This());
606+
}
607+
592608
NAN_METHOD(Database::LoadExtension) {
593609
NanScope();
594610
Database* db = ObjectWrap::Unwrap<Database>(args.This());
@@ -602,6 +618,16 @@ NAN_METHOD(Database::LoadExtension) {
602618
NanReturnValue(args.This());
603619
}
604620

621+
void Database::Work_BeginLoadCharacterTokenizer(Baton* baton) {
622+
assert(baton->db->locked);
623+
assert(baton->db->open);
624+
assert(baton->db->_handle);
625+
assert(baton->db->pending == 0);
626+
int status = uv_queue_work(uv_default_loop(),
627+
&baton->request, Work_LoadCharacterTokenizer, (uv_after_work_cb)Work_AfterLoadExtension);
628+
assert(status == 0);
629+
}
630+
605631
void Database::Work_BeginLoadExtension(Baton* baton) {
606632
assert(baton->db->locked);
607633
assert(baton->db->open);
@@ -612,6 +638,53 @@ void Database::Work_BeginLoadExtension(Baton* baton) {
612638
assert(status == 0);
613639
}
614640

641+
/*
642+
** Register a tokenizer implementation with FTS3 or FTS4.
643+
*/
644+
static int registerTokenizer(
645+
sqlite3 *db,
646+
char *zName,
647+
const sqlite3_tokenizer_module *p
648+
){
649+
int rc;
650+
sqlite3_stmt *pStmt;
651+
const char *zSql = "SELECT fts3_tokenizer(?, ?)";
652+
653+
rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
654+
if( rc!=SQLITE_OK ){
655+
return rc;
656+
}
657+
658+
sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
659+
sqlite3_bind_blob(pStmt, 2, &p, sizeof(p), SQLITE_STATIC);
660+
sqlite3_step(pStmt);
661+
662+
return sqlite3_finalize(pStmt);
663+
}
664+
665+
void Database::Work_LoadCharacterTokenizer(uv_work_t* req) {
666+
LoadExtensionBaton* baton = static_cast<LoadExtensionBaton*>(req->data);
667+
668+
sqlite3_enable_load_extension(baton->db->_handle, 1);
669+
670+
char* message = NULL;
671+
char token_name[] = "character";
672+
const sqlite3_tokenizer_module *ptr;
673+
674+
// get the tokenizer
675+
get_character_tokenizer_module(&ptr);
676+
677+
// register character tokenizer, note that you need to register it everytime the database is opened
678+
registerTokenizer(baton->db->_handle, token_name, ptr);
679+
680+
sqlite3_enable_load_extension(baton->db->_handle, 0);
681+
682+
if (baton->status != SQLITE_OK && message != NULL) {
683+
baton->message = std::string(message);
684+
sqlite3_free(message);
685+
}
686+
}
687+
615688
void Database::Work_LoadExtension(uv_work_t* req) {
616689
LoadExtensionBaton* baton = static_cast<LoadExtensionBaton*>(req->data);
617690

‎src/database.h‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -142,6 +142,11 @@ class Database : public ObjectWrap {
142142
static void Work_Close(uv_work_t* req);
143143
static void Work_AfterClose(uv_work_t* req);
144144

145+
static NAN_METHOD(LoadCharacterTokenizer);
146+
static void Work_BeginLoadCharacterTokenizer(Baton* baton);
147+
static void Work_LoadCharacterTokenizer(uv_work_t* req);
148+
static void Work_AfterLoadCharacterTokenizer(uv_work_t* req);
149+
145150
static NAN_METHOD(LoadExtension);
146151
static void Work_BeginLoadExtension(Baton* baton);
147152
static void Work_LoadExtension(uv_work_t* req);

0 commit comments

Comments
 (0)