/*

** 2014 Dec 01

**

** The author disclaims copyright to this source code. In place of

** a legal notice, here is a blessing:

**

** May you do good and not evil.

** May you find forgiveness for yourself and forgive others.

** May you share freely, never taking more than you give.

**

******************************************************************************

**

*/

#ifdef SQLITE_TEST

#include "tclsqlite.h"

#ifdef SQLITE_ENABLE_FTS5

#include "fts5.h"

#include <string.h>

#include <assert.h>

#include <stdlib.h>

#ifdef SQLITE_DEBUG

extern int sqlite3_fts5_may_be_corrupt;

#endif

extern int sqlite3Fts5TestRegisterMatchinfo(sqlite3*);

extern int sqlite3Fts5TestRegisterTok(sqlite3*, fts5_api*);

/*************************************************************************

** This is a copy of the first part of the SqliteDb structure in

** tclsqlite.c. We need it here so that the get_sqlite_pointer routine

** can extract the sqlite3* pointer from an existing Tcl SQLite

** connection.

*/

extern const char *sqlite3ErrName(int);

struct SqliteDb {

sqlite3 *db;

};

/*

** Decode a pointer to an sqlite3 object.

*/

static int f5tDbPointer(Tcl_Interp *interp, Tcl_Obj *pObj, sqlite3 **ppDb){

struct SqliteDb *p;

Tcl_CmdInfo cmdInfo;

char *z = Tcl_GetString(pObj);

if( Tcl_GetCommandInfo(interp, z, &cmdInfo) ){

p = (struct SqliteDb*)cmdInfo.objClientData;

*ppDb = p->db;

return TCL_OK;

}

return TCL_ERROR;

}

/* End of code that accesses the SqliteDb struct.

**************************************************************************/

static int f5tResultToErrorCode(const char *zRes){

struct ErrorCode {

int rc;

const char *zError;

} aErr[] = {

{ SQLITE_DONE, "SQLITE_DONE" },

{ SQLITE_ERROR, "SQLITE_ERROR" },

{ SQLITE_OK, "SQLITE_OK" },

{ SQLITE_OK, "" },

};

int i;

for(i=0; i<sizeof(aErr)/sizeof(aErr[0]); i++){

if( 0==sqlite3_stricmp(zRes, aErr[i].zError) ){

return aErr[i].rc;

}

}

return SQLITE_ERROR;

}

static int SQLITE_TCLAPI f5tDbAndApi(

Tcl_Interp *interp,

Tcl_Obj *pObj,

sqlite3 **ppDb,

fts5_api **ppApi

){

sqlite3 *db = 0;

int rc = f5tDbPointer(interp, pObj, &db);

if( rc!=TCL_OK ){

return TCL_ERROR;

}else{

sqlite3_stmt *pStmt = 0;

fts5_api *pApi = 0;

rc = sqlite3_prepare_v2(db, "SELECT fts5(?1)", -1, &pStmt, 0);

if( rc!=SQLITE_OK ){

Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), (char*)0);

return TCL_ERROR;

}

sqlite3_bind_pointer(pStmt, 1, (void*)&pApi, "fts5_api_ptr", 0);

sqlite3_step(pStmt);

if( sqlite3_finalize(pStmt)!=SQLITE_OK ){

Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), (char*)0);

return TCL_ERROR;

}

*ppDb = db;

*ppApi = pApi;

}

return TCL_OK;

}

typedef struct F5tFunction F5tFunction;

struct F5tFunction {

Tcl_Interp *interp;

Tcl_Obj *pScript;

};

typedef struct F5tApi F5tApi;

struct F5tApi {

const Fts5ExtensionApi *pApi;

Fts5Context *pFts;

};

/*

** An object of this type is used with the xSetAuxdata() and xGetAuxdata()

** API test wrappers. The tcl interface allows a single tcl value to be

** saved using xSetAuxdata(). Instead of simply storing a pointer to the

** tcl object, the code in this file wraps it in an sqlite3_malloc'd

** instance of the following struct so that if the destructor is not

** correctly invoked it will be reported as an SQLite memory leak.

*/

typedef struct F5tAuxData F5tAuxData;

struct F5tAuxData {

Tcl_Obj *pObj;

};

static int xTokenizeCb(

void *pCtx,

int tflags,

const char *zToken, int nToken,

int iStart, int iEnd

){

F5tFunction *p = (F5tFunction*)pCtx;

Tcl_Obj *pEval = Tcl_DuplicateObj(p->pScript);

int rc;

Tcl_IncrRefCount(pEval);

Tcl_ListObjAppendElement(p->interp, pEval, Tcl_NewStringObj(zToken, nToken));

Tcl_ListObjAppendElement(p->interp, pEval, Tcl_NewIntObj(iStart));

Tcl_ListObjAppendElement(p->interp, pEval, Tcl_NewIntObj(iEnd));

rc = Tcl_EvalObjEx(p->interp, pEval, 0);

Tcl_DecrRefCount(pEval);

if( rc==TCL_OK ){

rc = f5tResultToErrorCode(Tcl_GetStringResult(p->interp));

}

return rc;

}

static int SQLITE_TCLAPI xF5tApi(void*, Tcl_Interp*, int, Tcl_Obj *CONST []);

static int xQueryPhraseCb(

const Fts5ExtensionApi *pApi,

Fts5Context *pFts,

void *pCtx

){

F5tFunction *p = (F5tFunction*)pCtx;

static sqlite3_int64 iCmd = 0;

Tcl_Obj *pEval;

int rc;

char zCmd[64];

F5tApi sApi;

sApi.pApi = pApi;

sApi.pFts = pFts;

sprintf(zCmd, "f5t_2_%lld", iCmd++);

Tcl_CreateObjCommand(p->interp, zCmd, xF5tApi, &sApi, 0);

pEval = Tcl_DuplicateObj(p->pScript);

Tcl_IncrRefCount(pEval);

Tcl_ListObjAppendElement(p->interp, pEval, Tcl_NewStringObj(zCmd, -1));

rc = Tcl_EvalObjEx(p->interp, pEval, 0);

Tcl_DecrRefCount(pEval);

Tcl_DeleteCommand(p->interp, zCmd);

if( rc==TCL_OK ){

rc = f5tResultToErrorCode(Tcl_GetStringResult(p->interp));

}

return rc;

}

static void xSetAuxdataDestructor(void *p){

F5tAuxData *pData = (F5tAuxData*)p;

Tcl_DecrRefCount(pData->pObj);

sqlite3_free(pData);

}

/*

** api sub-command...

**

** Description...

*/

static int SQLITE_TCLAPI xF5tApi(

void * clientData,

Tcl_Interp *interp,

int objc,

Tcl_Obj *CONST objv[]

){

struct Sub {

const char *zName;

int nArg;

const char *zMsg;

} aSub[] = {

{ "xColumnCount", 0, "" }, /* 0 */

{ "xRowCount", 0, "" }, /* 1 */

{ "xColumnTotalSize", 1, "COL" }, /* 2 */

{ "xTokenize", 2, "TEXT SCRIPT" }, /* 3 */

{ "xPhraseCount", 0, "" }, /* 4 */

{ "xPhraseSize", 1, "PHRASE" }, /* 5 */

{ "xInstCount", 0, "" }, /* 6 */

{ "xInst", 1, "IDX" }, /* 7 */

{ "xRowid", 0, "" }, /* 8 */

{ "xColumnText", 1, "COL" }, /* 9 */

{ "xColumnSize", 1, "COL" }, /* 10 */

{ "xQueryPhrase", 2, "PHRASE SCRIPT" }, /* 11 */

{ "xSetAuxdata", 1, "VALUE" }, /* 12 */

{ "xGetAuxdata", 1, "CLEAR" }, /* 13 */

{ "xSetAuxdataInt", 1, "INTEGER" }, /* 14 */

{ "xGetAuxdataInt", 1, "CLEAR" }, /* 15 */

{ "xPhraseForeach", 4, "IPHRASE COLVAR OFFVAR SCRIPT" }, /* 16 */

{ "xPhraseColumnForeach", 3, "IPHRASE COLVAR SCRIPT" }, /* 17 */

{ "xQueryToken", 2, "IPHRASE ITERM" }, /* 18 */

{ "xInstToken", 2, "IDX ITERM" }, /* 19 */

{ "xColumnLocale", 1, "COL" }, /* 20 */

{ 0, 0, 0}

};

int rc;

int iSub = 0;

F5tApi *p = (F5tApi*)clientData;

if( objc<2 ){

Tcl_WrongNumArgs(interp, 1, objv, "SUB-COMMAND");

return TCL_ERROR;

}

rc = Tcl_GetIndexFromObjStruct(

interp, objv[1], aSub, sizeof(aSub[0]), "SUB-COMMAND", 0, &iSub

);

if( rc!=TCL_OK ) return rc;

if( aSub[iSub].nArg!=objc-2 ){

Tcl_WrongNumArgs(interp, 1, objv, aSub[iSub].zMsg);

return TCL_ERROR;

}

#define CASE(i,str) case i: assert( strcmp(aSub[i].zName, str)==0 );

switch( iSub ){

CASE(0, "xColumnCount") {

int nCol;

nCol = p->pApi->xColumnCount(p->pFts);

if( rc==SQLITE_OK ){

Tcl_SetObjResult(interp, Tcl_NewIntObj(nCol));

}

break;

}

CASE(1, "xRowCount") {

sqlite3_int64 nRow;

rc = p->pApi->xRowCount(p->pFts, &nRow);

if( rc==SQLITE_OK ){

Tcl_SetObjResult(interp, Tcl_NewWideIntObj(nRow));

}

break;

}

CASE(2, "xColumnTotalSize") {

int iCol;

sqlite3_int64 nSize;

if( Tcl_GetIntFromObj(interp, objv[2], &iCol) ) return TCL_ERROR;

rc = p->pApi->xColumnTotalSize(p->pFts, iCol, &nSize);

if( rc==SQLITE_OK ){

Tcl_SetObjResult(interp, Tcl_NewWideIntObj(nSize));

}

break;

}

CASE(3, "xTokenize") {

Tcl_Size nText;

char *zText = Tcl_GetStringFromObj(objv[2], &nText);

F5tFunction ctx;

ctx.interp = interp;

ctx.pScript = objv[3];

rc = p->pApi->xTokenize(p->pFts, zText, (int)nText, &ctx, xTokenizeCb);

if( rc==SQLITE_OK ){

Tcl_ResetResult(interp);

}

return rc;

}

CASE(4, "xPhraseCount") {

int nPhrase;

nPhrase = p->pApi->xPhraseCount(p->pFts);

if( rc==SQLITE_OK ){

Tcl_SetObjResult(interp, Tcl_NewIntObj(nPhrase));

}

break;

}

CASE(5, "xPhraseSize") {

int iPhrase;

int sz;

if( Tcl_GetIntFromObj(interp, objv[2], &iPhrase) ){

return TCL_ERROR;

}

sz = p->pApi->xPhraseSize(p->pFts, iPhrase);

if( rc==SQLITE_OK ){

Tcl_SetObjResult(interp, Tcl_NewIntObj(sz));

}

break;

}

CASE(6, "xInstCount") {

int nInst;

rc = p->pApi->xInstCount(p->pFts, &nInst);

if( rc==SQLITE_OK ){

Tcl_SetObjResult(interp, Tcl_NewIntObj(nInst));

}

break;

}

CASE(7, "xInst") {

int iIdx, ip, ic, io;

if( Tcl_GetIntFromObj(interp, objv[2], &iIdx) ){

return TCL_ERROR;

}

rc = p->pApi->xInst(p->pFts, iIdx, &ip, &ic, &io);

if( rc==SQLITE_OK ){

Tcl_Obj *pList = Tcl_NewObj();

Tcl_ListObjAppendElement(interp, pList, Tcl_NewIntObj(ip));

Tcl_ListObjAppendElement(interp, pList, Tcl_NewIntObj(ic));

Tcl_ListObjAppendElement(interp, pList, Tcl_NewIntObj(io));

Tcl_SetObjResult(interp, pList);

}

break;

}

CASE(8, "xRowid") {

sqlite3_int64 iRowid = p->pApi->xRowid(p->pFts);

Tcl_SetObjResult(interp, Tcl_NewWideIntObj(iRowid));

break;

}

CASE(9, "xColumnText") {

const char *z = 0;

int n = 0;

int iCol;

if( Tcl_GetIntFromObj(interp, objv[2], &iCol) ){

return TCL_ERROR;

}

rc = p->pApi->xColumnText(p->pFts, iCol, &z, &n);

if( rc==SQLITE_OK ){

Tcl_SetObjResult(interp, Tcl_NewStringObj(z, n));

}

break;

}

CASE(10, "xColumnSize") {

int n = 0;

int iCol;

if( Tcl_GetIntFromObj(interp, objv[2], &iCol) ){

return TCL_ERROR;

}

rc = p->pApi->xColumnSize(p->pFts, iCol, &n);

if( rc==SQLITE_OK ){

Tcl_SetObjResult(interp, Tcl_NewIntObj(n));

}

break;

}

CASE(11, "xQueryPhrase") {

int iPhrase;

F5tFunction ctx;

if( Tcl_GetIntFromObj(interp, objv[2], &iPhrase) ){

return TCL_ERROR;

}

ctx.interp = interp;

ctx.pScript = objv[3];

rc = p->pApi->xQueryPhrase(p->pFts, iPhrase, &ctx, xQueryPhraseCb);

if( rc==SQLITE_OK ){

Tcl_ResetResult(interp);

}

break;

}

CASE(12, "xSetAuxdata") {

F5tAuxData *pData = (F5tAuxData*)sqlite3_malloc(sizeof(F5tAuxData));

if( pData==0 ){

Tcl_AppendResult(interp, "out of memory", (char*)0);

return TCL_ERROR;

}

pData->pObj = objv[2];

Tcl_IncrRefCount(pData->pObj);

rc = p->pApi->xSetAuxdata(p->pFts, pData, xSetAuxdataDestructor);

break;

}

CASE(13, "xGetAuxdata") {

F5tAuxData *pData;

int bClear;

if( Tcl_GetBooleanFromObj(interp, objv[2], &bClear) ){

return TCL_ERROR;

}

pData = (F5tAuxData*)p->pApi->xGetAuxdata(p->pFts, bClear);

if( pData==0 ){

Tcl_ResetResult(interp);

}else{

Tcl_SetObjResult(interp, pData->pObj);

if( bClear ){

xSetAuxdataDestructor((void*)pData);

}

}

break;

}

/* These two - xSetAuxdataInt and xGetAuxdataInt - are similar to the

** xSetAuxdata and xGetAuxdata methods implemented above. The difference

** is that they may only save an integer value as auxiliary data, and

** do not specify a destructor function. */

CASE(14, "xSetAuxdataInt") {

int iVal;

if( Tcl_GetIntFromObj(interp, objv[2], &iVal) ) return TCL_ERROR;

rc = p->pApi->xSetAuxdata(p->pFts, (void*)((char*)0 + iVal), 0);

break;

}

CASE(15, "xGetAuxdataInt") {

int iVal;

int bClear;

if( Tcl_GetBooleanFromObj(interp, objv[2], &bClear) ) return TCL_ERROR;

iVal = (int)((char*)p->pApi->xGetAuxdata(p->pFts, bClear) - (char*)0);

Tcl_SetObjResult(interp, Tcl_NewIntObj(iVal));

break;

}

CASE(16, "xPhraseForeach") {

int iPhrase;

int iCol;

int iOff;

const char *zColvar;

const char *zOffvar;

Tcl_Obj *pScript = objv[5];

Fts5PhraseIter iter;

if( Tcl_GetIntFromObj(interp, objv[2], &iPhrase) ) return TCL_ERROR;

zColvar = Tcl_GetString(objv[3]);

zOffvar = Tcl_GetString(objv[4]);

rc = p->pApi->xPhraseFirst(p->pFts, iPhrase, &iter, &iCol, &iOff);

if( rc!=SQLITE_OK ){

Tcl_AppendResult(interp, sqlite3ErrName(rc), (char*)0);

return TCL_ERROR;

}

for( ;iCol>=0; p->pApi->xPhraseNext(p->pFts, &iter, &iCol, &iOff) ){

Tcl_SetVar2Ex(interp, zColvar, 0, Tcl_NewIntObj(iCol), 0);

Tcl_SetVar2Ex(interp, zOffvar, 0, Tcl_NewIntObj(iOff), 0);

rc = Tcl_EvalObjEx(interp, pScript, 0);

if( rc==TCL_CONTINUE ) rc = TCL_OK;

if( rc!=TCL_OK ){

if( rc==TCL_BREAK ) rc = TCL_OK;

break;

}

}

break;

}

CASE(17, "xPhraseColumnForeach") {

int iPhrase;

int iCol;

const char *zColvar;

Tcl_Obj *pScript = objv[4];

Fts5PhraseIter iter;

if( Tcl_GetIntFromObj(interp, objv[2], &iPhrase) ) return TCL_ERROR;

zColvar = Tcl_GetString(objv[3]);

rc = p->pApi->xPhraseFirstColumn(p->pFts, iPhrase, &iter, &iCol);

if( rc!=SQLITE_OK ){

Tcl_SetResult(interp, (char*)sqlite3ErrName(rc), TCL_VOLATILE);

return TCL_ERROR;

}

for( ; iCol>=0; p->pApi->xPhraseNextColumn(p->pFts, &iter, &iCol)){

Tcl_SetVar2Ex(interp, zColvar, 0, Tcl_NewIntObj(iCol), 0);

rc = Tcl_EvalObjEx(interp, pScript, 0);

if( rc==TCL_CONTINUE ) rc = TCL_OK;

if( rc!=TCL_OK ){

if( rc==TCL_BREAK ) rc = TCL_OK;

break;

}

}

break;

}

CASE(18, "xQueryToken") {

const char *pTerm = 0;

int nTerm = 0;

int iPhrase = 0;

int iTerm = 0;

if( Tcl_GetIntFromObj(interp, objv[2], &iPhrase) ) return TCL_ERROR;

if( Tcl_GetIntFromObj(interp, objv[3], &iTerm) ) return TCL_ERROR;

rc = p->pApi->xQueryToken(p->pFts, iPhrase, iTerm, &pTerm, &nTerm);

if( rc==SQLITE_OK ){

Tcl_SetObjResult(interp, Tcl_NewStringObj(pTerm, nTerm));

}

break;

}

CASE(19, "xInstToken") {

const char *pTerm = 0;

int nTerm = 0;

int iIdx = 0;

int iTerm = 0;

if( Tcl_GetIntFromObj(interp, objv[2], &iIdx) ) return TCL_ERROR;

if( Tcl_GetIntFromObj(interp, objv[3], &iTerm) ) return TCL_ERROR;

rc = p->pApi->xInstToken(p->pFts, iIdx, iTerm, &pTerm, &nTerm);

if( rc==SQLITE_OK ){

Tcl_SetObjResult(interp, Tcl_NewStringObj(pTerm, nTerm));

}

break;

}

CASE(20, "xColumnLocale") {

const char *z = 0;

int n = 0;

int iCol;

if( Tcl_GetIntFromObj(interp, objv[2], &iCol) ){

return TCL_ERROR;

}

rc = p->pApi->xColumnLocale(p->pFts, iCol, &z, &n);

if( rc==SQLITE_OK && z ){

Tcl_SetObjResult(interp, Tcl_NewStringObj(z, n));

}

break;

}

default:

assert( 0 );

break;

}

#undef CASE

if( rc!=SQLITE_OK ){

Tcl_SetResult(interp, (char*)sqlite3ErrName(rc), TCL_VOLATILE);

return TCL_ERROR;

}

return TCL_OK;

}

static void xF5tFunction(

const Fts5ExtensionApi *pApi, /* API offered by current FTS version */

Fts5Context *pFts, /* First arg to pass to pApi functions */

sqlite3_context *pCtx, /* Context for returning result/error */

int nVal, /* Number of values in apVal[] array */

sqlite3_value **apVal /* Array of trailing arguments */

){

F5tFunction *p = (F5tFunction*)pApi->xUserData(pFts);

Tcl_Obj *pEval; /* Script to evaluate */

int i;

int rc;

static sqlite3_int64 iCmd = 0;

char zCmd[64];

F5tApi sApi;

sApi.pApi = pApi;

sApi.pFts = pFts;

sprintf(zCmd, "f5t_%lld", iCmd++);

Tcl_CreateObjCommand(p->interp, zCmd, xF5tApi, &sApi, 0);

pEval = Tcl_DuplicateObj(p->pScript);

Tcl_IncrRefCount(pEval);

Tcl_ListObjAppendElement(p->interp, pEval, Tcl_NewStringObj(zCmd, -1));

for(i=0; i<nVal; i++){

Tcl_Obj *pObj = 0;

switch( sqlite3_value_type(apVal[i]) ){

case SQLITE_TEXT:

pObj = Tcl_NewStringObj((const char*)sqlite3_value_text(apVal[i]), -1);

break;

case SQLITE_BLOB:

pObj = Tcl_NewByteArrayObj(

sqlite3_value_blob(apVal[i]), sqlite3_value_bytes(apVal[i])

);

break;

case SQLITE_INTEGER:

pObj = Tcl_NewWideIntObj(sqlite3_value_int64(apVal[i]));

break;

case SQLITE_FLOAT:

pObj = Tcl_NewDoubleObj(sqlite3_value_double(apVal[i]));

break;

default:

pObj = Tcl_NewObj();

break;

}

Tcl_ListObjAppendElement(p->interp, pEval, pObj);

}

rc = Tcl_EvalObjEx(p->interp, pEval, TCL_GLOBAL_ONLY);

Tcl_DecrRefCount(pEval);

Tcl_DeleteCommand(p->interp, zCmd);

if( rc!=TCL_OK ){

sqlite3_result_error(pCtx, Tcl_GetStringResult(p->interp), -1);

}else{

Tcl_Obj *pVar = Tcl_GetObjResult(p->interp);

const char *zType = (pVar->typePtr ? pVar->typePtr->name : "");

char c = zType[0];

if( c=='b' && strcmp(zType,"bytearray")==0 && pVar->bytes==0 ){

/* Only return a BLOB type if the Tcl variable is a bytearray and

** has no string representation. */

Tcl_Size nn;

unsigned char *data = Tcl_GetByteArrayFromObj(pVar, &nn);

sqlite3_result_blob(pCtx, data, (int)nn, SQLITE_TRANSIENT);

}else if( c=='b' && strcmp(zType,"boolean")==0 ){

int n;

Tcl_GetIntFromObj(0, pVar, &n);

sqlite3_result_int(pCtx, n);

}else if( c=='d' && strcmp(zType,"double")==0 ){

double r;

Tcl_GetDoubleFromObj(0, pVar, &r);

sqlite3_result_double(pCtx, r);

}else if( (c=='w' && strcmp(zType,"wideInt")==0) ||

(c=='i' && strcmp(zType,"int")==0) ){

Tcl_WideInt v;

Tcl_GetWideIntFromObj(0, pVar, &v);

sqlite3_result_int64(pCtx, v);

}else{

Tcl_Size nn;

unsigned char *data = (unsigned char *)Tcl_GetStringFromObj(pVar, &nn);

sqlite3_result_text(pCtx, (char*)data, (int)nn, SQLITE_TRANSIENT);

}

}

}

static void xF5tDestroy(void *pCtx){

F5tFunction *p = (F5tFunction*)pCtx;

Tcl_DecrRefCount(p->pScript);

ckfree((char *)p);

}

/*

** sqlite3_fts5_create_function DB NAME SCRIPT

**

** Description...

*/

static int SQLITE_TCLAPI f5tCreateFunction(

void * clientData,

Tcl_Interp *interp,

int objc,

Tcl_Obj *CONST objv[]

){

char *zName;

Tcl_Obj *pScript;

sqlite3 *db = 0;

fts5_api *pApi = 0;

F5tFunction *pCtx = 0;

int rc;

if( objc!=4 ){

Tcl_WrongNumArgs(interp, 1, objv, "DB NAME SCRIPT");

return TCL_ERROR;

}

if( f5tDbAndApi(interp, objv[1], &db, &pApi) ) return TCL_ERROR;

zName = Tcl_GetString(objv[2]);

pScript = objv[3];

pCtx = (F5tFunction*)ckalloc(sizeof(F5tFunction));

pCtx->interp = interp;

pCtx->pScript = pScript;

Tcl_IncrRefCount(pScript);

rc = pApi->xCreateFunction(

pApi, zName, (void*)pCtx, xF5tFunction, xF5tDestroy

);

if( rc!=SQLITE_OK ){

Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), (char*)0);

return TCL_ERROR;

}

return TCL_OK;

}

typedef struct F5tTokenizeCtx F5tTokenizeCtx;

struct F5tTokenizeCtx {

Tcl_Obj *pRet;

int bSubst;

const char *zInput;

};

static int xTokenizeCb2(

void *pCtx,

int tflags,

const char *zToken, int nToken,

int iStart, int iEnd

){

F5tTokenizeCtx *p = (F5tTokenizeCtx*)pCtx;

if( p->bSubst ){

Tcl_ListObjAppendElement(0, p->pRet, Tcl_NewStringObj(zToken, nToken));

Tcl_ListObjAppendElement(

0, p->pRet, Tcl_NewStringObj(&p->zInput[iStart], iEnd-iStart)

);

}else{

Tcl_ListObjAppendElement(0, p->pRet, Tcl_NewStringObj(zToken, nToken));

Tcl_ListObjAppendElement(0, p->pRet, Tcl_NewIntObj(iStart));

Tcl_ListObjAppendElement(0, p->pRet, Tcl_NewIntObj(iEnd));

}

return SQLITE_OK;

}

/*

** sqlite3_fts5_tokenize DB TOKENIZER TEXT

**

** Description...

*/

static int SQLITE_TCLAPI f5tTokenize(

void * clientData,

Tcl_Interp *interp,

int objc,

Tcl_Obj *CONST objv[]

){

char *pCopy = 0;

char *zText = 0;

Tcl_Size nText = 0;

sqlite3 *db = 0;

fts5_api *pApi = 0;

Fts5Tokenizer *pTok = 0;

fts5_tokenizer tokenizer;

Tcl_Obj *pRet = 0;

void *pUserdata;

int rc;

Tcl_Size nArg;

const char **azArg;

F5tTokenizeCtx ctx;

if( objc!=4 && objc!=5 ){

Tcl_WrongNumArgs(interp, 1, objv, "?-subst? DB NAME TEXT");

return TCL_ERROR;

}

if( objc==5 ){

char *zOpt = Tcl_GetString(objv[1]);

if( strcmp("-subst", zOpt) ){

Tcl_AppendResult(interp, "unrecognized option: ", zOpt, (char*)0);

return TCL_ERROR;

}

}

if( f5tDbAndApi(interp, objv[objc-3], &db, &pApi) ) return TCL_ERROR;

if( Tcl_SplitList(interp, Tcl_GetString(objv[objc-2]), &nArg, &azArg) ){

return TCL_ERROR;

}

if( nArg==0 ){

Tcl_AppendResult(interp, "no such tokenizer: ", (char*)0);

Tcl_Free((void*)azArg);

return TCL_ERROR;

}

zText = Tcl_GetStringFromObj(objv[objc-1], &nText);

rc = pApi->xFindTokenizer(pApi, azArg[0], &pUserdata, &tokenizer);

if( rc!=SQLITE_OK ){

Tcl_AppendResult(interp, "no such tokenizer: ", azArg[0], (char*)0);

return TCL_ERROR;

}

rc = tokenizer.xCreate(pUserdata, &azArg[1], (int)(nArg-1), &pTok);

if( rc!=SQLITE_OK ){

Tcl_AppendResult(interp, "error in tokenizer.xCreate()", (char*)0);

return TCL_ERROR;

}

if( nText>0 ){

pCopy = sqlite3_malloc(nText);

if( pCopy==0 ){

tokenizer.xDelete(pTok);

Tcl_AppendResult(interp, "error in sqlite3_malloc()", (char*)0);

return TCL_ERROR;

}else{

memcpy(pCopy, zText, nText);

}

}

pRet = Tcl_NewObj();

Tcl_IncrRefCount(pRet);

ctx.bSubst = (objc==5);

ctx.pRet = pRet;

ctx.zInput = pCopy;

rc = tokenizer.xTokenize(

pTok, (void*)&ctx, FTS5_TOKENIZE_DOCUMENT, pCopy,(int)nText, xTokenizeCb2

);

tokenizer.xDelete(pTok);

sqlite3_free(pCopy);

if( rc!=SQLITE_OK ){

Tcl_AppendResult(interp, "error in tokenizer.xTokenize()", (char*)0);

Tcl_DecrRefCount(pRet);

return TCL_ERROR;

}

Tcl_Free((void*)azArg);

Tcl_SetObjResult(interp, pRet);

Tcl_DecrRefCount(pRet);

return TCL_OK;

}

/*************************************************************************

** Start of tokenizer wrapper.

*/

typedef struct F5tTokenizerContext F5tTokenizerContext;

typedef struct F5tTokenizerCb F5tTokenizerCb;

typedef struct F5tTokenizerModule F5tTokenizerModule;

typedef struct F5tTokenizerInstance F5tTokenizerInstance;

struct F5tTokenizerContext {

void *pCtx;

int (*xToken)(void*, int, const char*, int, int, int);

F5tTokenizerInstance *pInst;

};

struct F5tTokenizerModule {

Tcl_Interp *interp;

Tcl_Obj *pScript;

void *pParentCtx;

fts5_tokenizer_v2 parent_v2;

fts5_tokenizer parent;

F5tTokenizerContext *pContext;

};

/*

** zLocale:

** Within a call to xTokenize_v2(), pLocale/nLocale store the locale

** passed to the call by fts5. This can be retrieved by a Tcl tokenize

** script using [sqlite3_fts5_locale].

*/

struct F5tTokenizerInstance {

Tcl_Interp *interp;

Tcl_Obj *pScript;

F5tTokenizerModule *pModule;

Fts5Tokenizer *pParent;

F5tTokenizerContext *pContext;

const char *pLocale;

int nLocale;

};

static int f5tTokenizerCreate(

void *pCtx,

const char **azArg,

int nArg,

Fts5Tokenizer **ppOut

){

Fts5Tokenizer *pParent = 0;

F5tTokenizerModule *pMod = (F5tTokenizerModule*)pCtx;

Tcl_Obj *pEval;

int rc = TCL_OK;

int i;

assert( pMod->parent_v2.xCreate==0 || pMod->parent.xCreate==0 );

if( pMod->parent_v2.xCreate ){

rc = pMod->parent_v2.xCreate(pMod->pParentCtx, 0, 0, &pParent);

}

if( pMod->parent.xCreate ){

rc = pMod->parent.xCreate(pMod->pParentCtx, 0, 0, &pParent);

}

pEval = Tcl_DuplicateObj(pMod->pScript);

Tcl_IncrRefCount(pEval);

for(i=0; rc==TCL_OK && i<nArg; i++){

Tcl_Obj *pObj = Tcl_NewStringObj(azArg[i], -1);

rc = Tcl_ListObjAppendElement(pMod->interp, pEval, pObj);

}

if( rc==TCL_OK ){

rc = Tcl_EvalObjEx(pMod->interp, pEval, TCL_GLOBAL_ONLY);

}

Tcl_DecrRefCount(pEval);

if( rc==TCL_OK ){

F5tTokenizerInstance *pInst;

pInst = (F5tTokenizerInstance*)ckalloc(sizeof(F5tTokenizerInstance));

memset(pInst, 0, sizeof(F5tTokenizerInstance));

pInst->interp = pMod->interp;

pInst->pScript = Tcl_GetObjResult(pMod->interp);

pInst->pContext = pMod->pContext;

pInst->pParent = pParent;

pInst->pModule = pMod;

Tcl_IncrRefCount(pInst->pScript);

*ppOut = (Fts5Tokenizer*)pInst;

}

return rc;

}

static void f5tTokenizerDelete(Fts5Tokenizer *p){

F5tTokenizerInstance *pInst = (F5tTokenizerInstance*)p;

if( pInst ){

if( pInst->pParent ){

if( pInst->pModule->parent_v2.xDelete ){

pInst->pModule->parent_v2.xDelete(pInst->pParent);

}else{

pInst->pModule->parent.xDelete(pInst->pParent);

}

}

Tcl_DecrRefCount(pInst->pScript);

ckfree((char *)pInst);

}

}

static int f5tTokenizerReallyTokenize(

Fts5Tokenizer *p,

void *pCtx,

int flags,

const char *pText, int nText,

int (*xToken)(void*, int, const char*, int, int, int)

){

F5tTokenizerInstance *pInst = (F5tTokenizerInstance*)p;

F5tTokenizerInstance *pOldInst = 0;

void *pOldCtx;

int (*xOldToken)(void*, int, const char*, int, int, int);

Tcl_Obj *pEval;

int rc;

const char *zFlags;

pOldCtx = pInst->pContext->pCtx;

xOldToken = pInst->pContext->xToken;

pOldInst = pInst->pContext->pInst;

pInst->pContext->pCtx = pCtx;

pInst->pContext->xToken = xToken;

pInst->pContext->pInst = pInst;

assert(

flags==FTS5_TOKENIZE_DOCUMENT

|| flags==FTS5_TOKENIZE_AUX

|| flags==FTS5_TOKENIZE_QUERY

|| flags==(FTS5_TOKENIZE_QUERY | FTS5_TOKENIZE_PREFIX)

);

pEval = Tcl_DuplicateObj(pInst->pScript);

Tcl_IncrRefCount(pEval);

switch( flags ){

case FTS5_TOKENIZE_DOCUMENT:

zFlags = "document";

break;

case FTS5_TOKENIZE_AUX:

zFlags = "aux";

break;

case FTS5_TOKENIZE_QUERY:

zFlags = "query";

break;

case (FTS5_TOKENIZE_PREFIX | FTS5_TOKENIZE_QUERY):

zFlags = "prefixquery";

break;

default:

assert( 0 );

zFlags = "invalid";

break;

}

Tcl_ListObjAppendElement(pInst->interp, pEval, Tcl_NewStringObj(zFlags, -1));

Tcl_ListObjAppendElement(pInst->interp, pEval, Tcl_NewStringObj(pText,nText));

rc = Tcl_EvalObjEx(pInst->interp, pEval, TCL_GLOBAL_ONLY);

Tcl_DecrRefCount(pEval);

pInst->pContext->pCtx = pOldCtx;

pInst->pContext->xToken = xOldToken;

pInst->pContext->pInst = pOldInst;

return rc;

}

typedef struct CallbackCtx CallbackCtx;

struct CallbackCtx {

Fts5Tokenizer *p;

void *pCtx;

int flags;

int (*xToken)(void*, int, const char*, int, int, int);

};

static int f5tTokenizeCallback(

void *pCtx,

int tflags,

const char *z, int n,

int iStart, int iEnd

){

CallbackCtx *p = (CallbackCtx*)pCtx;

return f5tTokenizerReallyTokenize(p->p, p->pCtx, p->flags, z, n, p->xToken);

}

static int f5tTokenizerTokenize_v2(

Fts5Tokenizer *p,