From 24aaa8383a6bdde9ea1a8f1eb0375216ed638d8a Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Thu, 3 Mar 2022 16:46:44 -0500 Subject: [PATCH 001/231] relocate includes to crypto.h --- src/crypto.h | 18 ++++++++++++++++++ src/crypto_impl.c | 16 ---------------- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/src/crypto.h b/src/crypto.h index b443355a6..c45c48f6d 100644 --- a/src/crypto.h +++ b/src/crypto.h @@ -38,11 +38,29 @@ #include "sqliteInt.h" #include "btreeInt.h" #include "pager.h" +#include "vdbeInt.h" #ifdef __ANDROID__ #include #endif +#include + +#if defined(_WIN32) || defined(SQLITE_OS_WINRT) +#include /* amalgamator: dontcache */ +#else +#include /* amalgamator: dontcache */ +#endif + +#ifndef OMIT_MEMLOCK +#if defined(__unix__) || defined(__APPLE__) || defined(_AIX) +#include /* amalgamator: dontcache */ +#include /* amalgamator: dontcache */ +#include /* amalgamator: dontcache */ +#include /* amalgamator: dontcache */ +#endif +#endif + /* extensions defined in pager.c */ void *sqlite3PagerGetCodec(Pager*); void sqlite3PagerSetCodec(Pager*, void *(*)(void*,void*,Pgno,int), void (*)(void*,int,int), void (*)(void*), void *); diff --git a/src/crypto_impl.c b/src/crypto_impl.c index 5495d7e62..03322ef82 100644 --- a/src/crypto_impl.c +++ b/src/crypto_impl.c @@ -33,22 +33,6 @@ #include "sqlcipher.h" #include "crypto.h" -#include - -#if defined(_WIN32) || defined(SQLITE_OS_WINRT) -#include /* amalgamator: dontcache */ -#else -#include /* amalgamator: dontcache */ -#endif - -#ifndef OMIT_MEMLOCK -#if defined(__unix__) || defined(__APPLE__) || defined(_AIX) -#include /* amalgamator: dontcache */ -#include /* amalgamator: dontcache */ -#include /* amalgamator: dontcache */ -#include /* amalgamator: dontcache */ -#endif -#endif #ifdef SQLCIPHER_TEST static volatile unsigned int cipher_test_flags = 0; From 7a22a928400cdd3d5c7c90c0aa130e06b1a20111 Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Fri, 4 Mar 2022 16:15:54 -0500 Subject: [PATCH 002/231] support trunk rename of PAGER_MJ_PGNO to PAGER_SJ_PGNO --- src/crypto_impl.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/crypto_impl.c b/src/crypto_impl.c index 03322ef82..9566176d5 100644 --- a/src/crypto_impl.c +++ b/src/crypto_impl.c @@ -34,6 +34,10 @@ #include "sqlcipher.h" #include "crypto.h" +#ifndef PAGER_MJ_PGNO +#define PAGER_MJ_PGNO(x) PAGER_SJ_PGNO(x) +#endif + #ifdef SQLCIPHER_TEST static volatile unsigned int cipher_test_flags = 0; unsigned int sqlcipher_get_test_flags() { From 54bfd761a9ee0367551f22c690b0752bd271a4a2 Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Tue, 29 Mar 2022 09:30:33 -0400 Subject: [PATCH 003/231] include crypto.h for OPENSSL_VERSION_TEXT using BoringSSL --- src/crypto_openssl.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/crypto_openssl.c b/src/crypto_openssl.c index d8f223619..20b9af9c7 100644 --- a/src/crypto_openssl.c +++ b/src/crypto_openssl.c @@ -34,6 +34,7 @@ #include "sqliteInt.h" #include "crypto.h" #include "sqlcipher.h" +#include #include #include #include From 6ea9c7910b6425234d384dfa1c4b14a9847d1646 Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Mon, 4 Apr 2022 12:19:49 -0400 Subject: [PATCH 004/231] change compilation / amalgamation order of sqlcipher sources --- Makefile.in | 4 ++-- Makefile.msc | 18 +++++++++--------- tool/mksqlite3c.tcl | 18 +++++++++--------- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/Makefile.in b/Makefile.in index ad4be7b86..e56cb546b 100644 --- a/Makefile.in +++ b/Makefile.in @@ -228,7 +228,6 @@ LIBOBJ = $(LIBOBJS$(USE_AMALGAMATION)) # All of the source code files. # SRC = \ - $(CRYPTOSRC) \ $(TOP)/src/alter.c \ $(TOP)/src/analyze.c \ $(TOP)/src/attach.c \ @@ -331,7 +330,8 @@ SRC = \ $(TOP)/src/wherecode.c \ $(TOP)/src/whereexpr.c \ $(TOP)/src/whereInt.h \ - $(TOP)/src/window.c + $(TOP)/src/window.c \ + $(CRYPTOSRC) # Source code for extensions # diff --git a/Makefile.msc b/Makefile.msc index 49e7daa7f..5ef9a3dd7 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -1292,14 +1292,6 @@ LIBRESOBJS = # Core source code files, part 1. # SRC00 = \ - $(TOP)\src\crypto.c \ - $(TOP)\src\crypto_cc.c \ - $(TOP)\src\crypto_impl.c \ - $(TOP)\src\crypto_libtomcrypt.c \ - $(TOP)\src\crypto_nss.c \ - $(TOP)\src\crypto_openssl.c \ - $(TOP)\src\crypto.h \ - $(TOP)\src\sqlcipher.h \ $(TOP)\src\alter.c \ $(TOP)\src\analyze.c \ $(TOP)\src\attach.c \ @@ -1382,7 +1374,15 @@ SRC01 = \ $(TOP)\src\where.c \ $(TOP)\src\wherecode.c \ $(TOP)\src\whereexpr.c \ - $(TOP)\src\window.c + $(TOP)\src\window.c \ + $(TOP)\src\crypto.c \ + $(TOP)\src\crypto_cc.c \ + $(TOP)\src\crypto_impl.c \ + $(TOP)\src\crypto_libtomcrypt.c \ + $(TOP)\src\crypto_nss.c \ + $(TOP)\src\crypto_openssl.c \ + $(TOP)\src\crypto.h \ + $(TOP)\src\sqlcipher.h # Core miscellaneous files. # diff --git a/tool/mksqlite3c.tcl b/tool/mksqlite3c.tcl index ac12fe275..77d14ca80 100644 --- a/tool/mksqlite3c.tcl +++ b/tool/mksqlite3c.tcl @@ -128,8 +128,6 @@ close $in # text of the file in-line. The file only needs to be included once. # foreach hdr { - crypto.h - sqlcipher.h btree.h btreeInt.h fts3.h @@ -164,6 +162,8 @@ foreach hdr { vxworks.h wal.h whereInt.h + crypto.h + sqlcipher.h } { set available_hdr($hdr) 1 } @@ -332,13 +332,6 @@ foreach file { os_common.h ctime.c - crypto.c - crypto_impl.c - crypto_libtomcrypt.c - crypto_nss.c - crypto_openssl.c - crypto_cc.c - global.c status.c date.c @@ -450,6 +443,13 @@ foreach file { sqlite3session.c fts5.c stmt.c + + crypto.c + crypto_impl.c + crypto_libtomcrypt.c + crypto_nss.c + crypto_openssl.c + crypto_cc.c } { copy_file $srcdir/$file } From 0d5220a0e197f61ad25cd7e4500b77ad7aebb564 Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Mon, 4 Apr 2022 12:20:45 -0400 Subject: [PATCH 005/231] namespace changes and fixes for non-amalgamated builds --- src/attach.c | 10 +++++----- src/backup.c | 9 +++++---- src/crypto.c | 46 +++++++++++++++++++++++----------------------- src/crypto.h | 10 ++++++---- src/crypto_impl.c | 20 ++++++++++---------- src/pager.c | 13 +++---------- src/pager.h | 2 +- src/sqlcipher.h | 14 ++++++++------ src/vacuum.c | 4 ++-- src/wal.c | 4 ++-- 10 files changed, 65 insertions(+), 67 deletions(-) diff --git a/src/attach.c b/src/attach.c index a4d49abad..95e7b3123 100644 --- a/src/attach.c +++ b/src/attach.c @@ -201,8 +201,8 @@ static void attachFunc( /* BEGIN SQLCIPHER */ #ifdef SQLITE_HAS_CODEC if( rc==SQLITE_OK ){ - extern int sqlite3CodecAttach(sqlite3*, int, const void*, int); - extern void sqlite3CodecGetKey(sqlite3*, int, void**, int*); + extern int sqlcipherCodecAttach(sqlite3*, int, const void*, int); + extern void sqlcipherCodecGetKey(sqlite3*, int, void**, int*); int nKey; char *zKey; int t = sqlite3_value_type(argv[2]); @@ -217,16 +217,16 @@ static void attachFunc( case SQLITE_BLOB: nKey = sqlite3_value_bytes(argv[2]); zKey = (char *)sqlite3_value_blob(argv[2]); - rc = sqlite3CodecAttach(db, db->nDb-1, zKey, nKey); + rc = sqlcipherCodecAttach(db, db->nDb-1, zKey, nKey); break; case SQLITE_NULL: /* No key specified. Use the key from URI filename, or if none, ** use the key from the main database. */ if( sqlite3CodecQueryParameters(db, zName, zPath)==0 ){ - sqlite3CodecGetKey(db, 0, (void**)&zKey, &nKey); + sqlcipherCodecGetKey(db, 0, (void**)&zKey, &nKey); if( nKey || sqlite3BtreeGetRequestedReserve(db->aDb[0].pBt)>0 ){ - rc = sqlite3CodecAttach(db, db->nDb-1, zKey, nKey); + rc = sqlcipherCodecAttach(db, db->nDb-1, zKey, nKey); } } break; diff --git a/src/backup.c b/src/backup.c index f6b592881..13b1558c5 100644 --- a/src/backup.c +++ b/src/backup.c @@ -157,12 +157,12 @@ sqlite3_backup *sqlite3_backup_init( #ifdef SQLITE_HAS_CODEC { extern int sqlcipher_find_db_index(sqlite3*, const char*); - extern void sqlite3CodecGetKey(sqlite3*, int, void**, int*); + extern void sqlcipherCodecGetKey(sqlite3*, int, void**, int*); int srcNKey, destNKey; void *zKey; - sqlite3CodecGetKey(pSrcDb, sqlcipher_find_db_index(pSrcDb, zSrcDb), &zKey, &srcNKey); - sqlite3CodecGetKey(pDestDb, sqlcipher_find_db_index(pDestDb, zDestDb), &zKey, &destNKey); + sqlcipherCodecGetKey(pSrcDb, sqlcipher_find_db_index(pSrcDb, zSrcDb), &zKey, &srcNKey); + sqlcipherCodecGetKey(pDestDb, sqlcipher_find_db_index(pDestDb, zDestDb), &zKey, &destNKey); zKey = NULL; /* either both databases must be plaintext, or both must be encrypted */ @@ -258,6 +258,7 @@ static int backupOnePage( const i64 iEnd = (i64)iSrcPg*(i64)nSrcPgsz; /* BEGIN SQLCIPHER */ #ifdef SQLITE_HAS_CODEC + extern void *sqlcipherPagerGetCodec(Pager*); /* Use BtreeGetReserveNoMutex() for the source b-tree, as although it is ** guaranteed that the shared-mutex is held by this thread, handle ** p->pSrc may not actually be the owner. */ @@ -286,7 +287,7 @@ static int backupOnePage( /* Backup is not possible if the page size of the destination is changing ** and a codec is in use. */ - if( nSrcPgsz!=nDestPgsz && sqlite3PagerGetCodec(pDestPager)!=0 ){ + if( nSrcPgsz!=nDestPgsz && sqlcipherPagerGetCodec(pDestPager)!=0 ){ rc = SQLITE_READONLY; } diff --git a/src/crypto.c b/src/crypto.c index f4092c417..37bb54f84 100644 --- a/src/crypto.c +++ b/src/crypto.c @@ -78,7 +78,7 @@ static int codec_set_pass_key(sqlite3* db, int nDb, const void *zKey, int nKey, struct Db *pDb = &db->aDb[nDb]; sqlcipher_log(SQLCIPHER_LOG_DEBUG, "codec_set_pass_key: db=%p nDb=%d for_ctx=%d", db, nDb, for_ctx); if(pDb->pBt) { - codec_ctx *ctx = (codec_ctx*) sqlite3PagerGetCodec(pDb->pBt->pBt->pPager); + codec_ctx *ctx = (codec_ctx*) sqlcipherPagerGetCodec(pDb->pBt->pBt->pPager); if(ctx) { return sqlcipher_codec_ctx_set_pass(ctx, zKey, nKey, for_ctx); @@ -97,7 +97,7 @@ int sqlcipher_codec_pragma(sqlite3* db, int iDb, Parse *pParse, const char *zLef int rc; if(pDb->pBt) { - ctx = (codec_ctx*) sqlite3PagerGetCodec(pDb->pBt->pBt->pPager); + ctx = (codec_ctx*) sqlcipherPagerGetCodec(pDb->pBt->pBt->pPager); } if(sqlite3_stricmp(zLeft, "key") !=0 && sqlite3_stricmp(zLeft, "rekey") != 0) { @@ -822,10 +822,10 @@ static void sqlite3FreeCodecArg(void *pCodecArg) { sqlcipher_deactivate(); /* cleanup related structures, OpenSSL etc, when codec is detatched */ } -int sqlite3CodecAttach(sqlite3* db, int nDb, const void *zKey, int nKey) { +int sqlcipherCodecAttach(sqlite3* db, int nDb, const void *zKey, int nKey) { struct Db *pDb = &db->aDb[nDb]; - sqlcipher_log(SQLCIPHER_LOG_DEBUG, "sqlite3CodecAttach: db=%p, nDb=%d", db, nDb); + sqlcipher_log(SQLCIPHER_LOG_DEBUG, "sqlcipherCodecAttach: db=%p, nDb=%d", db, nDb); if(nKey && zKey && pDb->pBt) { int rc; @@ -836,12 +836,12 @@ int sqlite3CodecAttach(sqlite3* db, int nDb, const void *zKey, int nKey) { /* check if the sqlite3_file is open, and if not force handle to NULL */ if((fd = sqlite3PagerFile(pPager))->pMethods == 0) fd = NULL; - sqlcipher_log(SQLCIPHER_LOG_DEBUG, "sqlite3CodecAttach: calling sqlcipher_activate()"); + sqlcipher_log(SQLCIPHER_LOG_DEBUG, "sqlcipherCodecAttach: calling sqlcipher_activate()"); sqlcipher_activate(); /* perform internal initialization for sqlcipher */ - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlite3CodecAttach: entering database mutex %p", db->mutex); + sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipherCodecAttach: entering database mutex %p", db->mutex); sqlite3_mutex_enter(db->mutex); - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlite3CodecAttach: entered database mutex %p", db->mutex); + sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipherCodecAttach: entered database mutex %p", db->mutex); #ifdef SQLCIPHER_EXT if((rc = sqlite3_set_authorizer(db, sqlcipher_license_authorizer, db)) != SQLITE_OK) { @@ -851,44 +851,44 @@ int sqlite3CodecAttach(sqlite3* db, int nDb, const void *zKey, int nKey) { #endif /* point the internal codec argument against the contet to be prepared */ - sqlcipher_log(SQLCIPHER_LOG_DEBUG, "sqlite3CodecAttach: calling sqlcipher_codec_ctx_init()"); + sqlcipher_log(SQLCIPHER_LOG_DEBUG, "sqlcipherCodecAttach: calling sqlcipher_codec_ctx_init()"); rc = sqlcipher_codec_ctx_init(&ctx, pDb, pDb->pBt->pBt->pPager, zKey, nKey); if(rc != SQLITE_OK) { /* initialization failed, do not attach potentially corrupted context */ - sqlcipher_log(SQLCIPHER_LOG_ERROR, "sqlite3CodecAttach: context initialization failed forcing error state with rc=%d", rc); + sqlcipher_log(SQLCIPHER_LOG_ERROR, "sqlcipherCodecAttach: context initialization failed forcing error state with rc=%d", rc); /* force an error at the pager level, such that even the upstream caller ignores the return code the pager will be in an error state and will process no further operations */ sqlite3pager_error(pPager, rc); pDb->pBt->pBt->db->errCode = rc; - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlite3CodecAttach: leaving database mutex %p (early return on rc=%d)", db->mutex, rc); + sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipherCodecAttach: leaving database mutex %p (early return on rc=%d)", db->mutex, rc); sqlite3_mutex_leave(db->mutex); - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlite3CodecAttach: left database mutex %p (early return on rc=%d)", db->mutex, rc); + sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipherCodecAttach: left database mutex %p (early return on rc=%d)", db->mutex, rc); return rc; } - sqlcipher_log(SQLCIPHER_LOG_DEBUG, "sqlite3CodecAttach: calling sqlite3PagerSetCodec()"); - sqlite3PagerSetCodec(sqlite3BtreePager(pDb->pBt), sqlite3Codec, NULL, sqlite3FreeCodecArg, (void *) ctx); + sqlcipher_log(SQLCIPHER_LOG_DEBUG, "sqlcipherCodecAttach: calling sqlcipherPagerSetCodec()"); + sqlcipherPagerSetCodec(sqlite3BtreePager(pDb->pBt), sqlite3Codec, NULL, sqlite3FreeCodecArg, (void *) ctx); - sqlcipher_log(SQLCIPHER_LOG_DEBUG, "sqlite3CodecAttach: calling codec_set_btree_to_codec_pagesize()"); + sqlcipher_log(SQLCIPHER_LOG_DEBUG, "sqlcipherCodecAttach: calling codec_set_btree_to_codec_pagesize()"); codec_set_btree_to_codec_pagesize(db, pDb, ctx); /* force secure delete. This has the benefit of wiping internal data when deleted and also ensures that all pages are written to disk (i.e. not skipped by sqlite3PagerDontWrite optimizations) */ - sqlcipher_log(SQLCIPHER_LOG_DEBUG, "sqlite3CodecAttach: calling sqlite3BtreeSecureDelete()"); + sqlcipher_log(SQLCIPHER_LOG_DEBUG, "sqlcipherCodecAttach: calling sqlite3BtreeSecureDelete()"); sqlite3BtreeSecureDelete(pDb->pBt, 1); /* if fd is null, then this is an in-memory database and we dont' want to overwrite the AutoVacuum settings if not null, then set to the default */ if(fd != NULL) { - sqlcipher_log(SQLCIPHER_LOG_DEBUG, "sqlite3CodecAttach: calling sqlite3BtreeSetAutoVacuum()"); + sqlcipher_log(SQLCIPHER_LOG_DEBUG, "sqlcipherCodecAttach: calling sqlite3BtreeSetAutoVacuum()"); sqlite3BtreeSetAutoVacuum(pDb->pBt, SQLITE_DEFAULT_AUTOVACUUM); } - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlite3CodecAttach: leaving database mutex %p", db->mutex); + sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipherCodecAttach: leaving database mutex %p", db->mutex); sqlite3_mutex_leave(db->mutex); - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlite3CodecAttach: left database mutex %p", db->mutex); + sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipherCodecAttach: left database mutex %p", db->mutex); } return SQLITE_OK; } @@ -921,7 +921,7 @@ int sqlite3_key_v2(sqlite3 *db, const char *zDb, const void *pKey, int nKey) { /* attach key if db and pKey are not null and nKey is > 0 */ if(db && pKey && nKey) { int db_index = sqlcipher_find_db_index(db, zDb); - return sqlite3CodecAttach(db, db_index, pKey, nKey); + return sqlcipherCodecAttach(db, db_index, pKey, nKey); } sqlcipher_log(SQLCIPHER_LOG_ERROR, "sqlite3_key_v2: no key provided"); return SQLITE_ERROR; @@ -955,7 +955,7 @@ int sqlite3_rekey_v2(sqlite3 *db, const char *zDb, const void *pKey, int nKey) { PgHdr *page; Pager *pPager = pDb->pBt->pBt->pPager; - ctx = (codec_ctx*) sqlite3PagerGetCodec(pDb->pBt->pBt->pPager); + ctx = (codec_ctx*) sqlcipherPagerGetCodec(pDb->pBt->pBt->pPager); if(ctx == NULL) { /* there was no codec attached to this database, so this should do nothing! */ @@ -1013,11 +1013,11 @@ int sqlite3_rekey_v2(sqlite3 *db, const char *zDb, const void *pKey, int nKey) { return SQLITE_ERROR; } -void sqlite3CodecGetKey(sqlite3* db, int nDb, void **zKey, int *nKey) { +void sqlcipherCodecGetKey(sqlite3* db, int nDb, void **zKey, int *nKey) { struct Db *pDb = &db->aDb[nDb]; - sqlcipher_log(SQLCIPHER_LOG_DEBUG, "sqlite3CodecGetKey:db=%p, nDb=%d", db, nDb); + sqlcipher_log(SQLCIPHER_LOG_DEBUG, "sqlcipherCodecGetKey:db=%p, nDb=%d", db, nDb); if( pDb->pBt ) { - codec_ctx *ctx = (codec_ctx*) sqlite3PagerGetCodec(pDb->pBt->pBt->pPager); + codec_ctx *ctx = (codec_ctx*) sqlcipherPagerGetCodec(pDb->pBt->pBt->pPager); if(ctx) { /* pass back the keyspec from the codec, unless PRAGMA cipher_store_pass diff --git a/src/crypto.h b/src/crypto.h index c45c48f6d..7d59159fa 100644 --- a/src/crypto.h +++ b/src/crypto.h @@ -61,9 +61,11 @@ #endif #endif +#include "sqlcipher.h" + /* extensions defined in pager.c */ -void *sqlite3PagerGetCodec(Pager*); -void sqlite3PagerSetCodec(Pager*, void *(*)(void*,void*,Pgno,int), void (*)(void*,int,int), void (*)(void*), void *); +void *sqlcipherPagerGetCodec(Pager*); +void sqlcipherPagerSetCodec(Pager*, void *(*)(void*,void*,Pgno,int), void (*)(void*,int,int), void (*)(void*), void *); int sqlite3pager_is_mj_pgno(Pager*, Pgno); void sqlite3pager_error(Pager*, int); void sqlite3pager_reset(Pager *pPager); @@ -227,8 +229,8 @@ typedef struct { /* crypto.c functions */ int sqlcipher_codec_pragma(sqlite3*, int, Parse*, const char *, const char*); -int sqlite3CodecAttach(sqlite3*, int, const void *, int); -void sqlite3CodecGetKey(sqlite3*, int, void**, int*); +int sqlcipherCodecAttach(sqlite3*, int, const void *, int); +void sqlcipherCodecGetKey(sqlite3*, int, void**, int*); void sqlcipher_exportFunc(sqlite3_context *, int, sqlite3_value **); /* crypto_impl.c functions */ diff --git a/src/crypto_impl.c b/src/crypto_impl.c index 9566176d5..8cae4ca62 100644 --- a/src/crypto_impl.c +++ b/src/crypto_impl.c @@ -287,8 +287,8 @@ void sqlcipher_deactivate() { optimized out by the compiler. Note: As suggested by Joachim Schipper (joachim.schipper@fox-it.com) */ -void* sqlcipher_memset(void *v, unsigned char value, u64 len) { - u64 i = 0; +void* sqlcipher_memset(void *v, unsigned char value, sqlite_uint64 len) { + sqlite_uint64 i = 0; volatile unsigned char *a = v; if (v == NULL) return v; @@ -304,9 +304,9 @@ void* sqlcipher_memset(void *v, unsigned char value, u64 len) { /* constant time memory check tests every position of a memory segement matches a single value (i.e. the memory is all zeros) returns 0 if match, 1 of no match */ -int sqlcipher_ismemset(const void *v, unsigned char value, u64 len) { +int sqlcipher_ismemset(const void *v, unsigned char value, sqlite_uint64 len) { const unsigned char *a = v; - u64 i = 0, result = 0; + sqlite_uint64 i = 0, result = 0; for(i = 0; i < len; i++) { result |= a[i] ^ value; @@ -328,7 +328,7 @@ int sqlcipher_memcmp(const void *v0, const void *v1, int len) { return (result != 0); } -void sqlcipher_mlock(void *ptr, u64 sz) { +void sqlcipher_mlock(void *ptr, sqlite_uint64 sz) { #ifndef OMIT_MEMLOCK #if defined(__unix__) || defined(__APPLE__) int rc; @@ -355,7 +355,7 @@ void sqlcipher_mlock(void *ptr, u64 sz) { #endif } -void sqlcipher_munlock(void *ptr, u64 sz) { +void sqlcipher_munlock(void *ptr, sqlite_uint64 sz) { #ifndef OMIT_MEMLOCK #if defined(__unix__) || defined(__APPLE__) int rc; @@ -390,7 +390,7 @@ void sqlcipher_munlock(void *ptr, u64 sz) { * If sz is > 0, and not compiled with OMIT_MEMLOCK, system will attempt to unlock the * memory segment so it can be paged */ -void sqlcipher_free(void *ptr, u64 sz) { +void sqlcipher_free(void *ptr, sqlite_uint64 sz) { sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipher_free: calling sqlcipher_memset(%p,0,%llu)", ptr, sz); sqlcipher_memset(ptr, 0, sz); sqlcipher_munlock(ptr, sz); @@ -402,7 +402,7 @@ void sqlcipher_free(void *ptr, u64 sz) { * reference counted and leak detection works. Unless compiled with OMIT_MEMLOCK * attempts to lock the memory pages so sensitive information won't be swapped */ -void* sqlcipher_malloc(u64 sz) { +void* sqlcipher_malloc(sqlite_uint64 sz) { void *ptr; sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipher_malloc: calling sqlite3Malloc(%llu)", sz); ptr = sqlite3Malloc(sz); @@ -1538,8 +1538,8 @@ int sqlcipher_codec_ctx_migrate(codec_ctx *ctx) { sqlcipher_log(SQLCIPHER_LOG_DEBUG, "set btree page size to %d res %d rc %d", default_page_size, nRes, rc); if( rc!=SQLITE_OK ) goto handle_error; - sqlite3CodecGetKey(db, db->nDb - 1, (void**)&keyspec, &keyspec_sz); - sqlite3CodecAttach(db, 0, keyspec, keyspec_sz); + sqlcipherCodecGetKey(db, db->nDb - 1, (void**)&keyspec, &keyspec_sz); + sqlcipherCodecAttach(db, 0, keyspec, keyspec_sz); srcfile = sqlite3PagerFile(pSrc->pBt->pPager); destfile = sqlite3PagerFile(pDest->pBt->pPager); diff --git a/src/pager.c b/src/pager.c index 2bfd8480e..a0e782f4b 100644 --- a/src/pager.c +++ b/src/pager.c @@ -7162,7 +7162,7 @@ const char *sqlite3PagerJournalname(Pager *pPager){ /* ** Set or retrieve the codec for this pager */ -void sqlite3PagerSetCodec( +void sqlcipherPagerSetCodec( Pager *pPager, void *(*xCodec)(void*,void*,Pgno,int), void (*xCodecSizeChng)(void*,int,int), @@ -7181,7 +7181,7 @@ void sqlite3PagerSetCodec( setGetterMethod(pPager); pagerReportSize(pPager); } -void *sqlite3PagerGetCodec(Pager *pPager){ +void *sqlcipherPagerGetCodec(Pager *pPager){ return pPager->pCodec; } @@ -7192,18 +7192,11 @@ void *sqlite3PagerGetCodec(Pager *pPager){ ** This function returns a pointer to a buffer containing the encrypted ** page content. If a malloc fails, this function may return NULL. */ -void *sqlite3PagerCodec(PgHdr *pPg){ +void *sqlcipherPagerCodec(PgHdr *pPg){ void *aData = 0; CODEC2(pPg->pPager, pPg->pData, pPg->pgno, 6, return 0, aData); return aData; } - -/* -** Return the current pager state -*/ -int sqlite3PagerState(Pager *pPager){ - return pPager->eState; -} #endif /* SQLITE_HAS_CODEC */ /* END SQLCIPHER */ diff --git a/src/pager.h b/src/pager.h index 4aecfae03..769cf290c 100644 --- a/src/pager.h +++ b/src/pager.h @@ -231,7 +231,7 @@ void sqlite3PagerRekey(DbPage*, Pgno, u16); /* BEGIN SQLCIPHER */ #if defined(SQLITE_HAS_CODEC) && !defined(SQLITE_OMIT_WAL) -void *sqlite3PagerCodec(DbPage *); +void *sqlcipherPagerCodec(DbPage *); #endif /* END SQLCIPHER */ diff --git a/src/sqlcipher.h b/src/sqlcipher.h index 048427225..545a05d7f 100644 --- a/src/sqlcipher.h +++ b/src/sqlcipher.h @@ -35,6 +35,8 @@ #ifndef SQLCIPHER_H #define SQLCIPHER_H +#include "sqlite3.h" + #define SQLCIPHER_HMAC_SHA1 0 #define SQLCIPHER_HMAC_SHA1_LABEL "HMAC_SHA1" #define SQLCIPHER_HMAC_SHA256 1 @@ -72,13 +74,13 @@ typedef struct { } sqlcipher_provider; /* utility functions */ -void* sqlcipher_malloc(u64); -void sqlcipher_mlock(void *, u64); -void sqlcipher_munlock(void *, u64); -void* sqlcipher_memset(void *, unsigned char, u64); -int sqlcipher_ismemset(const void *, unsigned char, u64); +void* sqlcipher_malloc(sqlite_uint64); +void sqlcipher_mlock(void *, sqlite_uint64); +void sqlcipher_munlock(void *, sqlite_uint64); +void* sqlcipher_memset(void *, unsigned char, sqlite_uint64); +int sqlcipher_ismemset(const void *, unsigned char, sqlite_uint64); int sqlcipher_memcmp(const void *, const void *, int); -void sqlcipher_free(void *, u64); +void sqlcipher_free(void *, sqlite_uint64); char* sqlcipher_version(); /* provider interfaces */ diff --git a/src/vacuum.c b/src/vacuum.c index ead60e78e..61b21e491 100644 --- a/src/vacuum.c +++ b/src/vacuum.c @@ -239,10 +239,10 @@ SQLITE_NOINLINE int sqlite3RunVacuum( /* BEGIN SQLCIPHER */ #ifdef SQLITE_HAS_CODEC if( db->nextPagesize ){ - extern void sqlite3CodecGetKey(sqlite3*, int, void**, int*); + extern void sqlcipherCodecGetKey(sqlite3*, int, void**, int*); int nKey; char *zKey; - sqlite3CodecGetKey(db, iDb, (void**)&zKey, &nKey); + sqlcipherCodecGetKey(db, iDb, (void**)&zKey, &nKey); if( nKey ) db->nextPagesize = 0; } #endif diff --git a/src/wal.c b/src/wal.c index 7cfbd79a9..113883cfb 100644 --- a/src/wal.c +++ b/src/wal.c @@ -3534,7 +3534,7 @@ static int walWriteOneFrame( void *pData; /* Data actually written */ u8 aFrame[WAL_FRAME_HDRSIZE]; /* Buffer to assemble frame-header in */ #if defined(SQLITE_HAS_CODEC) - if( (pData = sqlite3PagerCodec(pPage))==0 ) return SQLITE_NOMEM_BKPT; + if( (pData = sqlcipherPagerCodec(pPage))==0 ) return SQLITE_NOMEM_BKPT; #else pData = pPage->pData; #endif @@ -3721,7 +3721,7 @@ int sqlite3WalFrames( pWal->iReCksum = iWrite; } #if defined(SQLITE_HAS_CODEC) - if( (pData = sqlite3PagerCodec(p))==0 ) return SQLITE_NOMEM; + if( (pData = sqlcipherPagerCodec(p))==0 ) return SQLITE_NOMEM; #else pData = p->pData; #endif From 45e9c1f0285475744424bea8f971f2ffa2030f7d Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Mon, 4 Apr 2022 12:38:53 -0400 Subject: [PATCH 006/231] simplify conditional HMAC code for openssl versions and fix unfreed allocation reported in #424 --- src/crypto_openssl.c | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/src/crypto_openssl.c b/src/crypto_openssl.c index 20b9af9c7..0ac53dfb0 100644 --- a/src/crypto_openssl.c +++ b/src/crypto_openssl.c @@ -180,6 +180,7 @@ static int sqlcipher_openssl_random (void *ctx, void *buffer, int length) { static int sqlcipher_openssl_hmac(void *ctx, int algorithm, unsigned char *hmac_key, int key_sz, unsigned char *in, int in_sz, unsigned char *in2, int in2_sz, unsigned char *out) { int rc = 0; + #if (defined(OPENSSL_VERSION_NUMBER) && OPENSSL_VERSION_NUMBER < 0x30000000L) unsigned int outlen; HMAC_CTX* hctx = NULL; @@ -240,6 +241,15 @@ static int sqlcipher_openssl_hmac(void *ctx, int algorithm, unsigned char *hmac_ goto error; } + rc = SQLITE_OK; + goto cleanup; + +error: + rc = SQLITE_ERROR; + +cleanup: + if(hctx) HMAC_CTX_free(hctx); + #else size_t outlen; EVP_MAC *mac = NULL; @@ -317,17 +327,18 @@ static int sqlcipher_openssl_hmac(void *ctx, int algorithm, unsigned char *hmac_ goto error; } -#endif rc = SQLITE_OK; goto cleanup; + error: rc = SQLITE_ERROR; + cleanup: -#if (defined(OPENSSL_VERSION_NUMBER) && OPENSSL_VERSION_NUMBER < 0x30000000L) - if(hctx) HMAC_CTX_free(hctx); -#else if(hctx) EVP_MAC_CTX_free(hctx); + if(mac) EVP_MAC_free(mac); + #endif + return rc; } From 269be0c9a9bf765a62d4756e69f3d5f420dc5239 Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Mon, 4 Apr 2022 13:19:03 -0400 Subject: [PATCH 007/231] Revert "change compilation / amalgamation order of sqlcipher sources" This reverts commit 6ea9c7910b6425234d384dfa1c4b14a9847d1646. --- Makefile.in | 4 ++-- Makefile.msc | 18 +++++++++--------- tool/mksqlite3c.tcl | 18 +++++++++--------- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/Makefile.in b/Makefile.in index e56cb546b..ad4be7b86 100644 --- a/Makefile.in +++ b/Makefile.in @@ -228,6 +228,7 @@ LIBOBJ = $(LIBOBJS$(USE_AMALGAMATION)) # All of the source code files. # SRC = \ + $(CRYPTOSRC) \ $(TOP)/src/alter.c \ $(TOP)/src/analyze.c \ $(TOP)/src/attach.c \ @@ -330,8 +331,7 @@ SRC = \ $(TOP)/src/wherecode.c \ $(TOP)/src/whereexpr.c \ $(TOP)/src/whereInt.h \ - $(TOP)/src/window.c \ - $(CRYPTOSRC) + $(TOP)/src/window.c # Source code for extensions # diff --git a/Makefile.msc b/Makefile.msc index 5ef9a3dd7..49e7daa7f 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -1292,6 +1292,14 @@ LIBRESOBJS = # Core source code files, part 1. # SRC00 = \ + $(TOP)\src\crypto.c \ + $(TOP)\src\crypto_cc.c \ + $(TOP)\src\crypto_impl.c \ + $(TOP)\src\crypto_libtomcrypt.c \ + $(TOP)\src\crypto_nss.c \ + $(TOP)\src\crypto_openssl.c \ + $(TOP)\src\crypto.h \ + $(TOP)\src\sqlcipher.h \ $(TOP)\src\alter.c \ $(TOP)\src\analyze.c \ $(TOP)\src\attach.c \ @@ -1374,15 +1382,7 @@ SRC01 = \ $(TOP)\src\where.c \ $(TOP)\src\wherecode.c \ $(TOP)\src\whereexpr.c \ - $(TOP)\src\window.c \ - $(TOP)\src\crypto.c \ - $(TOP)\src\crypto_cc.c \ - $(TOP)\src\crypto_impl.c \ - $(TOP)\src\crypto_libtomcrypt.c \ - $(TOP)\src\crypto_nss.c \ - $(TOP)\src\crypto_openssl.c \ - $(TOP)\src\crypto.h \ - $(TOP)\src\sqlcipher.h + $(TOP)\src\window.c # Core miscellaneous files. # diff --git a/tool/mksqlite3c.tcl b/tool/mksqlite3c.tcl index 77d14ca80..ac12fe275 100644 --- a/tool/mksqlite3c.tcl +++ b/tool/mksqlite3c.tcl @@ -128,6 +128,8 @@ close $in # text of the file in-line. The file only needs to be included once. # foreach hdr { + crypto.h + sqlcipher.h btree.h btreeInt.h fts3.h @@ -162,8 +164,6 @@ foreach hdr { vxworks.h wal.h whereInt.h - crypto.h - sqlcipher.h } { set available_hdr($hdr) 1 } @@ -332,6 +332,13 @@ foreach file { os_common.h ctime.c + crypto.c + crypto_impl.c + crypto_libtomcrypt.c + crypto_nss.c + crypto_openssl.c + crypto_cc.c + global.c status.c date.c @@ -443,13 +450,6 @@ foreach file { sqlite3session.c fts5.c stmt.c - - crypto.c - crypto_impl.c - crypto_libtomcrypt.c - crypto_nss.c - crypto_openssl.c - crypto_cc.c } { copy_file $srcdir/$file } From ddd3ee442ae71c0d9ae99285f6ed860bdc5a90bc Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Mon, 4 Apr 2022 13:58:37 -0400 Subject: [PATCH 008/231] relocate crypto after vdbe --- tool/mksqlite3c.tcl | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tool/mksqlite3c.tcl b/tool/mksqlite3c.tcl index ac12fe275..a6549f6e8 100644 --- a/tool/mksqlite3c.tcl +++ b/tool/mksqlite3c.tcl @@ -332,13 +332,6 @@ foreach file { os_common.h ctime.c - crypto.c - crypto_impl.c - crypto_libtomcrypt.c - crypto_nss.c - crypto_openssl.c - crypto_cc.c - global.c status.c date.c @@ -389,6 +382,13 @@ foreach file { vdbevtab.c memjournal.c + crypto.c + crypto_impl.c + crypto_libtomcrypt.c + crypto_nss.c + crypto_openssl.c + crypto_cc.c + walker.c resolve.c expr.c From 7778787c61cccbdd2e18c891668f18f54a9d9edb Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Mon, 4 Apr 2022 14:20:20 -0400 Subject: [PATCH 009/231] define wrapping PAGER_MJ_PGNO for pager --- src/pager.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/pager.c b/src/pager.c index a0e782f4b..2e6fa231d 100644 --- a/src/pager.c +++ b/src/pager.c @@ -7882,6 +7882,10 @@ int sqlite3PagerWalFramesize(Pager *pPager){ /* BEGIN SQLCIPHER */ #ifdef SQLITE_HAS_CODEC +#ifndef PAGER_MJ_PGNO +#define PAGER_MJ_PGNO(x) PAGER_SJ_PGNO(x) +#endif + int sqlite3pager_is_mj_pgno(Pager *pPager, Pgno pgno) { return (PAGER_MJ_PGNO(pPager) == pgno) ? 1 : 0; } From 45475c423b50420e57ab763ab100c5deb919d0d1 Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Fri, 8 Apr 2022 12:39:49 -0400 Subject: [PATCH 010/231] fix issue where memory PRAGMA cipher_memory_security would report OFF when it was actually ON --- src/crypto_impl.c | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/src/crypto_impl.c b/src/crypto_impl.c index 8cae4ca62..f28c315da 100644 --- a/src/crypto_impl.c +++ b/src/crypto_impl.c @@ -74,9 +74,9 @@ static volatile int default_page_size = 4096; static volatile int default_plaintext_header_sz = 0; static volatile int default_hmac_algorithm = SQLCIPHER_HMAC_SHA512; static volatile int default_kdf_algorithm = SQLCIPHER_PBKDF2_HMAC_SHA512; -static volatile int mem_security_on = 0; -static volatile int mem_security_initialized = 0; -static volatile int mem_security_activated = 0; +static volatile int sqlcipher_mem_security_on = 0; +static volatile int sqlcipher_mem_executed = 0; +static volatile int sqlcipher_mem_initialized = 0; static volatile unsigned int sqlcipher_activate_count = 0; static volatile sqlite3_mem_methods default_mem_methods; static sqlcipher_provider *default_provider = NULL; @@ -99,10 +99,10 @@ static void sqlcipher_mem_shutdown(void *pAppData) { } static void *sqlcipher_mem_malloc(int n) { void *ptr = default_mem_methods.xMalloc(n); - if(mem_security_on) { + if(!sqlcipher_mem_executed) sqlcipher_mem_executed = 1; + if(sqlcipher_mem_security_on) { sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipher_mem_malloc: calling sqlcipher_mlock(%p,%d)", ptr, n); sqlcipher_mlock(ptr, n); - if(!mem_security_activated) mem_security_activated = 1; } return ptr; } @@ -111,19 +111,19 @@ static int sqlcipher_mem_size(void *p) { } static void sqlcipher_mem_free(void *p) { int sz; - if(mem_security_on) { + if(!sqlcipher_mem_executed) sqlcipher_mem_executed = 1; + if(sqlcipher_mem_security_on) { sz = sqlcipher_mem_size(p); sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipher_mem_free: calling sqlcipher_memset(%p,0,%d) and sqlcipher_munlock(%p, %d)", p, sz, p, sz); sqlcipher_memset(p, 0, sz); sqlcipher_munlock(p, sz); - if(!mem_security_activated) mem_security_activated = 1; } default_mem_methods.xFree(p); } static void *sqlcipher_mem_realloc(void *p, int n) { void *new = NULL; int orig_sz = 0; - if(mem_security_on) { + if(sqlcipher_mem_security_on) { orig_sz = sqlcipher_mem_size(p); if (n==0) { sqlcipher_mem_free(p); @@ -161,12 +161,13 @@ static sqlite3_mem_methods sqlcipher_mem_methods = { }; void sqlcipher_init_memmethods() { - if(mem_security_initialized) return; + if(sqlcipher_mem_initialized) return; if(sqlite3_config(SQLITE_CONFIG_GETMALLOC, &default_mem_methods) != SQLITE_OK || sqlite3_config(SQLITE_CONFIG_MALLOC, &sqlcipher_mem_methods) != SQLITE_OK) { - mem_security_on = mem_security_activated = 0; + sqlcipher_mem_security_on = sqlcipher_mem_executed = sqlcipher_mem_initialized = 0; + } else { + sqlcipher_mem_initialized = 1; } - mem_security_initialized = 1; } int sqlcipher_register_provider(sqlcipher_provider *p) { @@ -870,13 +871,16 @@ int sqlcipher_get_default_pagesize() { void sqlcipher_set_mem_security(int on) { /* memory security can only be enabled, not disabled */ if(on) { - mem_security_on = on; - mem_security_activated = 0; + sqlcipher_log(SQLCIPHER_LOG_DEBUG, "sqlcipher_set_mem_security: on"); + sqlcipher_mem_security_on = on; } } int sqlcipher_get_mem_security() { - return mem_security_on && mem_security_activated; + /* only report that memory security is enabled if pragma cipher_memory_security is ON and + SQLCipher's allocator/deallocator was run at least one timecurrently used */ + sqlcipher_log(SQLCIPHER_LOG_DEBUG, "sqlcipher_get_mem_security: sqlcipher_mem_security_on = %d, sqlcipher_mem_executed = %d", sqlcipher_mem_security_on, sqlcipher_mem_executed); + return sqlcipher_mem_security_on && sqlcipher_mem_executed; } From 5ddd6e7978085b6ae8d8264cb8d1863abd84638e Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Thu, 14 Apr 2022 15:50:45 -0400 Subject: [PATCH 011/231] Snapshot of upstream SQLite 3.38.2 --- Makefile.in | 17 +- Makefile.msc | 31 +- README.md | 8 +- VERSION | 2 +- autoconf/Makefile.msc | 9 +- autoconf/README.txt | 2 +- autoconf/configure.ac | 15 - configure | 60 +- configure.ac | 25 +- doc/json-enhancements.md | 144 ++++ ext/expert/sqlite3expert.c | 35 +- ext/fts3/fts3.c | 45 +- ext/fts3/fts3Int.h | 2 +- ext/fts3/fts3_tokenize_vtab.c | 6 +- ext/fts3/mkfts3amal.tcl | 115 ---- ext/fts5/fts5_index.c | 2 +- ext/fts5/fts5_main.c | 2 +- ext/fts5/test/fts5corrupt3.test | 154 +++++ ext/fts5/test/fts5eb.test | 2 +- ext/fts5/test/fts5integrity.test | 4 +- ext/lsm1/lsm_vtab.c | 2 +- ext/misc/csv.c | 2 +- ext/misc/fileio.c | 3 +- ext/misc/ieee754.c | 2 +- ext/misc/qpvtab.c | 461 +++++++++++++ ext/misc/regexp.c | 10 +- ext/misc/series.c | 2 +- ext/misc/sha1.c | 2 +- ext/misc/shathree.c | 1 + ext/misc/zipfile.c | 9 +- ext/rbu/sqlite3rbu.c | 2 +- ext/repair/sqlite3_checker.c.in | 1 - ext/rtree/geopoly.c | 11 +- ext/rtree/rtreeA.test | 6 +- ext/rtree/rtreecheck.test | 4 +- ext/rtree/test_rtreedoc.c | 2 +- ext/session/session8.test | 2 +- main.mk | 22 +- manifest | 387 +++++------ manifest.uuid | 2 +- src/alter.c | 98 ++- src/attach.c | 2 +- src/backup.c | 5 +- src/btree.c | 239 ++++--- src/btreeInt.h | 4 +- src/build.c | 61 +- src/callback.c | 1 - src/ctime.c | 23 +- src/date.c | 312 ++++++--- src/dbpage.c | 1 + src/delete.c | 20 +- src/expr.c | 31 +- src/fkey.c | 2 +- src/func.c | 124 ++-- src/global.c | 1 + src/insert.c | 27 +- ext/misc/json1.c => src/json.c | 402 +++++------ src/loadext.c | 7 + src/main.c | 72 +- src/malloc.c | 11 +- src/memjournal.c | 24 +- src/os_unix.c | 12 +- src/os_win.c | 6 +- src/pager.c | 43 +- src/parse.y | 36 +- src/pcache.c | 3 +- src/pragma.c | 4 + src/prepare.c | 66 +- src/printf.c | 58 +- src/resolve.c | 70 +- src/select.c | 107 +-- src/shell.c.in | 1102 +++++++++++++++++++++++------- src/sqlite.h.in | 374 +++++++++- src/sqlite3ext.h | 14 + src/sqliteInt.h | 59 +- src/tclsqlite.c | 48 +- src/test1.c | 187 ++++- src/test_bestindex.c | 195 +++++- src/test_config.c | 2 +- src/test_func.c | 6 +- src/tokenize.c | 47 +- src/treeview.c | 2 +- src/trigger.c | 64 +- src/update.c | 14 +- src/util.c | 51 +- src/vdbe.c | 218 ++++-- src/vdbeInt.h | 32 +- src/vdbeapi.c | 69 +- src/vdbeaux.c | 44 +- src/vdbeblob.c | 11 +- src/vdbemem.c | 11 +- src/vdbesort.c | 3 +- src/vtab.c | 19 +- src/wal.c | 6 +- src/where.c | 865 ++++++++++++++++++----- src/whereInt.h | 34 +- src/wherecode.c | 264 +++++-- src/whereexpr.c | 241 ++++++- src/window.c | 14 +- test/alterauth2.test | 1 + test/altercol.test | 73 +- test/altermalloc2.test | 15 +- test/altermalloc3.test | 8 +- test/alterqf.test | 10 +- test/altertab2.test | 2 +- test/atof1.test | 5 - test/auth.test | 43 +- test/autoindex4.test | 93 +++ test/backup.test | 9 + test/bestindex1.test | 24 +- test/bestindex2.test | 5 +- test/bestindex3.test | 5 +- test/bestindex4.test | 11 +- test/bestindex5.test | 7 +- test/bestindex6.test | 6 +- test/bestindex7.test | 6 +- test/bestindex8.test | 463 +++++++++++++ test/bind2.test | 63 ++ test/capi3d.test | 86 +++ test/check.test | 4 + test/collate1.test | 1 + test/ctime.test | 1 + test/date.test | 185 ++--- test/date3.test | 149 ++++ test/dbpage.test | 6 + test/e_createtable.test | 4 +- test/e_expr.test | 11 +- test/e_fkey.test | 4 +- test/eval.test | 2 +- test/expr.test | 1 + test/fts3aj.test | 20 +- test/fts3ak.test | 26 +- test/fts3corrupt.test | 2 +- test/fts3dropmod.test | 44 ++ test/fts4noti.test | 4 +- test/func.test | 29 +- test/func6.test | 9 + test/fuzz-oss1.test | 2 + test/fuzz.test | 2 +- test/fuzzcheck.c | 12 + test/fuzzdata8.db | Bin 3019776 -> 3283968 bytes test/in.test | 5 +- test/index.test | 2 +- test/insert4.test | 8 +- test/join.test | 1 + test/join5.test | 68 ++ test/json101.test | 13 +- test/json102.test | 61 +- test/json103.test | 5 - test/json104.test | 5 - test/json105.test | 5 - test/merge1.test | 143 ++++ test/misc1.test | 2 +- test/pager1.test | 28 + test/printf2.test | 19 +- test/quote.test | 1 + test/releasetest_data.tcl | 16 +- test/returning1.test | 43 ++ test/rowvalue.test | 33 + test/rowvalue5.test | 4 +- test/select6.test | 1 - test/shell1.test | 32 +- test/shell2.test | 2 +- test/shell3.test | 4 +- test/shell4.test | 8 + test/shell5.test | 89 ++- test/strict1.test | 27 + test/subquery.test | 21 +- test/swarmvtab.test | 12 +- test/tabfunc01.test | 45 ++ test/tclsqlite.test | 9 +- test/tkt-7bbfb7d442.test | 2 +- test/tkt3442.test | 1 + test/tkt3841.test | 1 + test/trigger1.test | 11 + test/update.test | 2 +- test/upfrom2.test | 20 + test/upsert2.test | 8 +- test/utf16align.test | 1 + test/vtab6.test | 1 + test/vtabdistinct.test | 41 ++ test/vtabrhs1.test | 76 +++ test/walro2.test | 689 +++++++++---------- test/where7.test | 24 + test/window6.test | 1 + test/without_rowid1.test | 23 +- tool/lemon.c | 2 - tool/logest.c | 8 +- tool/mkctimec.tcl | 125 ++-- tool/mksqlite3c.tcl | 2 +- tool/omittest.tcl | 1 + tool/sqldiff.c | 48 +- tool/vdbe-compress.tcl | 2 +- tool/warnings.sh | 4 +- 194 files changed, 7996 insertions(+), 2743 deletions(-) create mode 100644 doc/json-enhancements.md delete mode 100644 ext/fts3/mkfts3amal.tcl create mode 100644 ext/misc/qpvtab.c rename ext/misc/json1.c => src/json.c (90%) create mode 100644 test/bestindex8.test create mode 100644 test/bind2.test create mode 100644 test/date3.test create mode 100644 test/fts3dropmod.test create mode 100644 test/merge1.test create mode 100644 test/vtabdistinct.test create mode 100644 test/vtabrhs1.test mode change 100644 => 100755 tool/mkctimec.tcl diff --git a/Makefile.in b/Makefile.in index b3ea4628b..e5d30d030 100644 --- a/Makefile.in +++ b/Makefile.in @@ -180,7 +180,7 @@ LIBOBJS0 = alter.lo analyze.lo attach.lo auth.lo \ fts3_unicode.lo fts3_unicode2.lo fts3_write.lo \ fts5.lo \ func.lo global.lo hash.lo \ - icu.lo insert.lo json1.lo legacy.lo loadext.lo \ + icu.lo insert.lo json.lo legacy.lo loadext.lo \ main.lo malloc.lo mem0.lo mem1.lo mem2.lo mem3.lo mem5.lo \ memdb.lo memjournal.lo \ mutex.lo mutex_noop.lo mutex_unix.lo mutex_w32.lo \ @@ -234,6 +234,7 @@ SRC = \ $(TOP)/src/hash.h \ $(TOP)/src/hwtime.h \ $(TOP)/src/insert.c \ + $(TOP)/src/json.c \ $(TOP)/src/legacy.c \ $(TOP)/src/loadext.c \ $(TOP)/src/main.c \ @@ -366,7 +367,6 @@ SRC += \ $(TOP)/ext/rbu/sqlite3rbu.h \ $(TOP)/ext/rbu/sqlite3rbu.c SRC += \ - $(TOP)/ext/misc/json1.c \ $(TOP)/ext/misc/stmt.c # Generated source code files @@ -460,6 +460,7 @@ TESTSRC += \ $(TOP)/ext/misc/normalize.c \ $(TOP)/ext/misc/percentile.c \ $(TOP)/ext/misc/prefixes.c \ + $(TOP)/ext/misc/qpvtab.c \ $(TOP)/ext/misc/regexp.c \ $(TOP)/ext/misc/remember.c \ $(TOP)/ext/misc/series.c \ @@ -608,7 +609,7 @@ TESTOPTS = --verbose=file --output=test-out.txt # Extra compiler options for various shell tools # -SHELL_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS4 +SHELL_OPT = -DSQLITE_ENABLE_FTS4 #SHELL_OPT += -DSQLITE_ENABLE_FTS5 SHELL_OPT += -DSQLITE_ENABLE_RTREE SHELL_OPT += -DSQLITE_ENABLE_EXPLAIN_COMMENTS @@ -618,8 +619,8 @@ SHELL_OPT += -DSQLITE_ENABLE_DBPAGE_VTAB SHELL_OPT += -DSQLITE_ENABLE_DBSTAT_VTAB SHELL_OPT += -DSQLITE_ENABLE_BYTECODE_VTAB SHELL_OPT += -DSQLITE_ENABLE_OFFSET_SQL_FUNC -FUZZERSHELL_OPT = -DSQLITE_ENABLE_JSON1 -FUZZCHECK_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ +FUZZERSHELL_OPT = +FUZZCHECK_OPT = -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ FUZZCHECK_OPT += -DSQLITE_MAX_MEMORY=50000000 FUZZCHECK_OPT += -DSQLITE_PRINTF_PRECISION_LIMIT=1000 FUZZCHECK_OPT += -DSQLITE_ENABLE_FTS4 @@ -873,6 +874,9 @@ hash.lo: $(TOP)/src/hash.c $(HDR) insert.lo: $(TOP)/src/insert.c $(HDR) $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/insert.c +json.lo: $(TOP)/src/json.c + $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/json.c + legacy.lo: $(TOP)/src/legacy.c $(HDR) $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/legacy.c @@ -1176,9 +1180,6 @@ userauth.lo: $(TOP)/ext/userauth/userauth.c $(HDR) $(EXTHDR) sqlite3session.lo: $(TOP)/ext/session/sqlite3session.c $(HDR) $(EXTHDR) $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/session/sqlite3session.c -json1.lo: $(TOP)/ext/misc/json1.c - $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/misc/json1.c - stmt.lo: $(TOP)/ext/misc/stmt.c $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/misc/stmt.c diff --git a/Makefile.msc b/Makefile.msc index 3cbf560d7..d67b7f0b8 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -363,7 +363,6 @@ SQLITE_TCL_DEP = OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS3=1 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_RTREE=1 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_GEOPOLY=1 -OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_JSON1=1 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_STMTVTAB=1 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DBPAGE_VTAB=1 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DBSTAT_VTAB=1 @@ -580,17 +579,17 @@ RCC = $(RC) -DSQLITE_OS_WIN=1 -I. -I$(TOP) -I$(TOP)\src $(RCOPTS) $(RCCOPTS) # !IF $(USE_STDCALL)!=0 || $(FOR_WIN10)!=0 !IF "$(PLATFORM)"=="x86" -CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall -SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall +CORE_CCONV_OPTS = -Gz -guard:cf -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall +SHELL_CCONV_OPTS = -Gz -guard:cf -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall # <> -TEST_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall -DINCLUDE_SQLITE_TCL_H=1 -DSQLITE_TCLAPI=__cdecl +TEST_CCONV_OPTS = -Gz -guard:cf -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall -DINCLUDE_SQLITE_TCL_H=1 -DSQLITE_TCLAPI=__cdecl # <> !ELSE !IFNDEF PLATFORM -CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall -SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall +CORE_CCONV_OPTS = -Gz -guard:cf -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall +SHELL_CCONV_OPTS = -Gz -guard:cf -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall # <> -TEST_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall -DINCLUDE_SQLITE_TCL_H=1 -DSQLITE_TCLAPI=__cdecl +TEST_CCONV_OPTS = -Gz -guard:cf -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall -DINCLUDE_SQLITE_TCL_H=1 -DSQLITE_TCLAPI=__cdecl # <> !ELSE CORE_CCONV_OPTS = @@ -1248,7 +1247,7 @@ LIBOBJS0 = vdbe.lo parse.lo alter.lo analyze.lo attach.lo auth.lo \ fts3_tokenize_vtab.lo fts3_unicode.lo fts3_unicode2.lo fts3_write.lo \ fts5.lo \ func.lo global.lo hash.lo \ - icu.lo insert.lo json1.lo legacy.lo loadext.lo \ + icu.lo insert.lo json.lo legacy.lo loadext.lo \ main.lo malloc.lo mem0.lo mem1.lo mem2.lo mem3.lo mem5.lo \ memdb.lo memjournal.lo \ mutex.lo mutex_noop.lo mutex_unix.lo mutex_w32.lo \ @@ -1315,6 +1314,7 @@ SRC00 = \ $(TOP)\src\global.c \ $(TOP)\src\hash.c \ $(TOP)\src\insert.c \ + $(TOP)\src\json.c \ $(TOP)\src\legacy.c \ $(TOP)\src\loadext.c \ $(TOP)\src\main.c \ @@ -1445,7 +1445,6 @@ SRC07 = \ $(TOP)\ext\rtree\rtree.c \ $(TOP)\ext\session\sqlite3session.c \ $(TOP)\ext\rbu\sqlite3rbu.c \ - $(TOP)\ext\misc\json1.c \ $(TOP)\ext\misc\stmt.c # Extension header files, part 1. @@ -1579,6 +1578,7 @@ TESTEXT = \ $(TOP)\ext\misc\normalize.c \ $(TOP)\ext\misc\percentile.c \ $(TOP)\ext\misc\prefixes.c \ + $(TOP)\ext\misc\qpvtab.c \ $(TOP)\ext\misc\regexp.c \ $(TOP)\ext\misc\remember.c \ $(TOP)\ext\misc\series.c \ @@ -1695,9 +1695,9 @@ SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_OFFSET_SQL_FUNC=1 # <> # Extra compiler options for various test tools. # -MPTESTER_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS5 -FUZZERSHELL_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1 -FUZZCHECK_OPTS = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ -DSQLITE_MAX_MEMORY=50000000 -DSQLITE_PRINTF_PRECISION_LIMIT=1000 +MPTESTER_COMPILE_OPTS = -DSQLITE_ENABLE_FTS5 +FUZZERSHELL_COMPILE_OPTS = +FUZZCHECK_OPTS = -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ -DSQLITE_MAX_MEMORY=50000000 -DSQLITE_PRINTF_PRECISION_LIMIT=1000 FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_FTS4 FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_FTS5 FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_RTREE @@ -1990,6 +1990,9 @@ hash.lo: $(TOP)\src\hash.c $(HDR) insert.lo: $(TOP)\src\insert.c $(HDR) $(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\insert.c +json.lo: $(TOP)\src\json.c $(HDR) + $(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\json.c + legacy.lo: $(TOP)\src\legacy.c $(HDR) $(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\legacy.c @@ -2298,9 +2301,6 @@ fts3_unicode2.lo: $(TOP)\ext\fts3\fts3_unicode2.c $(HDR) $(EXTHDR) fts3_write.lo: $(TOP)\ext\fts3\fts3_write.c $(HDR) $(EXTHDR) $(LTCOMPILE) $(CORE_COMPILE_OPTS) $(NO_WARN) -DSQLITE_CORE -c $(TOP)\ext\fts3\fts3_write.c -json1.lo: $(TOP)\ext\misc\json1.c $(HDR) $(EXTHDR) - $(LTCOMPILE) $(CORE_COMPILE_OPTS) $(NO_WARN) -DSQLITE_CORE -c $(TOP)\ext\misc\json1.c - stmt.lo: $(TOP)\ext\misc\stmt.c $(HDR) $(EXTHDR) $(LTCOMPILE) $(CORE_COMPILE_OPTS) $(NO_WARN) -DSQLITE_CORE -c $(TOP)\ext\misc\stmt.c @@ -2392,7 +2392,6 @@ TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_DEFAULT_PAGE_SIZE=1024 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_STMTVTAB=1 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_DBPAGE_VTAB=1 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_BYTECODE_VTAB=1 -TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_JSON1=1 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_CKSUMVFS_STATIC=1 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) $(TEST_CCONV_OPTS) diff --git a/README.md b/README.md index b79fc67f1..9e8bb2610 100644 --- a/README.md +++ b/README.md @@ -309,9 +309,11 @@ describes its purpose and role within the larger system. The `manifest` file at the root directory of the source tree contains either a SHA3-256 hash (for newer files) or a SHA1 hash (for older files) for every source file in the repository. -The SHA3-256 hash of the `manifest` -file itself is the official name of the version of the source tree that you -have. The `manifest.uuid` file should contain the SHA3-256 hash of the +The name of the version of the entire source tree is just the +SHA3-256 hash of the `manifest` file itself, possibly with the +last line of that file omitted if the last line begins with +"`# Remove this line`". +The `manifest.uuid` file should contain the SHA3-256 hash of the `manifest` file. If all of the above hash comparisons are correct, then you can be confident that your source tree is authentic and unadulterated. diff --git a/VERSION b/VERSION index 8587f0520..9c4d7c1ff 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.37.2 +3.38.2 diff --git a/autoconf/Makefile.msc b/autoconf/Makefile.msc index 40d0e8113..e36eb21ea 100644 --- a/autoconf/Makefile.msc +++ b/autoconf/Makefile.msc @@ -285,7 +285,6 @@ SQLITE3EXEPDB = /pdb:sqlite3sh.pdb OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS3=1 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_RTREE=1 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_GEOPOLY=1 -OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_JSON1=1 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_STMTVTAB=1 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DBPAGE_VTAB=1 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DBSTAT_VTAB=1 @@ -502,12 +501,12 @@ RCC = $(RC) -DSQLITE_OS_WIN=1 -I. -I$(TOP) $(RCOPTS) $(RCCOPTS) # !IF $(USE_STDCALL)!=0 || $(FOR_WIN10)!=0 !IF "$(PLATFORM)"=="x86" -CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall -SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall +CORE_CCONV_OPTS = -Gz -guard:cf -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall +SHELL_CCONV_OPTS = -Gz -guard:cf -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall !ELSE !IFNDEF PLATFORM -CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall -SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall +CORE_CCONV_OPTS = -Gz -guard:cf -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall +SHELL_CCONV_OPTS = -Gz -guard:cf -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall !ELSE CORE_CCONV_OPTS = SHELL_CCONV_OPTS = diff --git a/autoconf/README.txt b/autoconf/README.txt index 6e62a4e13..ccf5e235a 100644 --- a/autoconf/README.txt +++ b/autoconf/README.txt @@ -105,7 +105,7 @@ may be specified in this manner as some require the amalgamation to be built with them enabled (see http://www.sqlite.org/compile.html). For example, the following will work: - "OPTS=-DSQLITE_ENABLE_STAT4=1 -DSQLITE_ENABLE_JSON1=1" + "OPTS=-DSQLITE_ENABLE_STAT4=1 -DSQLITE_OMIT_JSON=1" However, the following will not compile unless the amalgamation was built with it enabled: diff --git a/autoconf/configure.ac b/autoconf/configure.ac index e050786bd..0c7a32db1 100644 --- a/autoconf/configure.ac +++ b/autoconf/configure.ac @@ -173,21 +173,6 @@ else fi #----------------------------------------------------------------------- -#----------------------------------------------------------------------- -# --enable-json1 -# -AC_ARG_ENABLE(json1, [AS_HELP_STRING( - [--enable-json1], [include json1 support [default=yes]])], - [],[enable_json1=yes]) -AC_MSG_CHECKING([JSON functions]) -if test x"$enable_json1" = "xyes"; then - BUILD_CFLAGS="$BUILD_CFLAGS -DSQLITE_ENABLE_JSON1" - AC_MSG_RESULT([enabled]) -else - AC_MSG_RESULT([disabled]) -fi -#----------------------------------------------------------------------- - #----------------------------------------------------------------------- # --enable-rtree # diff --git a/configure b/configure index 55852f3ea..30c2e50c2 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for sqlite 3.37.2. +# Generated by GNU Autoconf 2.69 for sqlite 3.38.2. # # # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc. @@ -726,8 +726,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='sqlite' PACKAGE_TARNAME='sqlite' -PACKAGE_VERSION='3.37.2' -PACKAGE_STRING='sqlite 3.37.2' +PACKAGE_VERSION='3.38.2' +PACKAGE_STRING='sqlite 3.38.2' PACKAGE_BUGREPORT='' PACKAGE_URL='' @@ -905,13 +905,13 @@ enable_debug enable_amalgamation enable_load_extension enable_math +enable_json enable_all enable_memsys5 enable_memsys3 enable_fts3 enable_fts4 enable_fts5 -enable_json1 enable_update_limit enable_geopoly enable_rtree @@ -1468,7 +1468,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures sqlite 3.37.2 to adapt to many kinds of systems. +\`configure' configures sqlite 3.38.2 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1533,7 +1533,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of sqlite 3.37.2:";; + short | recursive ) echo "Configuration of sqlite 3.38.2:";; esac cat <<\_ACEOF @@ -1560,13 +1560,13 @@ Optional Features: --disable-load-extension Disable loading of external extensions --disable-math Disable math functions - --enable-all Enable FTS4, FTS5, Geopoly, JSON, RTree, Sessions + --disable-json Disable JSON functions + --enable-all Enable FTS4, FTS5, Geopoly, RTree, Sessions --enable-memsys5 Enable MEMSYS5 --enable-memsys3 Enable MEMSYS3 --enable-fts3 Enable the FTS3 extension --enable-fts4 Enable the FTS4 extension --enable-fts5 Enable the FTS5 extension - --enable-json1 Enable the JSON1 extension --enable-update-limit Enable the UPDATE/DELETE LIMIT clause --enable-geopoly Enable the GEOPOLY extension --enable-rtree Enable the RTREE extension @@ -1661,7 +1661,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -sqlite configure 3.37.2 +sqlite configure 3.38.2 generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. @@ -2080,7 +2080,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by sqlite $as_me 3.37.2, which was +It was created by sqlite $as_me 3.38.2, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ @@ -11497,6 +11497,24 @@ fi fi +########## +# Do we want to support JSON functions +# +# Check whether --enable-json was given. +if test "${enable_json+set}" = set; then : + enableval=$enable_json; +fi + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to support JSON functions" >&5 +$as_echo_n "checking whether to support JSON functions... " >&6; } +if test "$enable_json" = "no"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_OMIT_JSON" +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } +fi ######## # The --enable-all argument is short-hand to enable @@ -11701,24 +11719,6 @@ else $as_echo "no" >&6; } fi -######### -# See whether we should enable JSON1 -# Check whether --enable-json1 was given. -if test "${enable_json1+set}" = set; then : - enableval=$enable_json1; -fi - -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to support JSON" >&5 -$as_echo_n "checking whether to support JSON... " >&6; } -if test "${enable_json1}" = "yes" -o "${enable_all}" = "yes" ; then - OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_JSON1" - { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 -$as_echo "yes" >&6; } -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } -fi - ######### # See whether we should enable the LIMIT clause on UPDATE and DELETE # statements. @@ -12390,7 +12390,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by sqlite $as_me 3.37.2, which was +This file was extended by sqlite $as_me 3.38.2, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -12456,7 +12456,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -sqlite config.status 3.37.2 +sqlite config.status 3.38.2 configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" diff --git a/configure.ac b/configure.ac index 32fe0d229..cc805a00d 100644 --- a/configure.ac +++ b/configure.ac @@ -605,12 +605,24 @@ else AC_SEARCH_LIBS(ceil, m) fi +########## +# Do we want to support JSON functions +# +AC_ARG_ENABLE(json, +AC_HELP_STRING([--disable-json],[Disable JSON functions])) +AC_MSG_CHECKING([whether to support JSON functions]) +if test "$enable_json" = "no"; then + AC_MSG_RESULT([no]) + OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_OMIT_JSON" +else + AC_MSG_RESULT([yes]) +fi ######## # The --enable-all argument is short-hand to enable # multiple extensions. AC_ARG_ENABLE(all, AC_HELP_STRING([--enable-all], - [Enable FTS4, FTS5, Geopoly, JSON, RTree, Sessions])) + [Enable FTS4, FTS5, Geopoly, RTree, Sessions])) ########## # Do we want to support memsys3 and/or memsys5 @@ -666,17 +678,6 @@ else AC_MSG_RESULT([no]) fi -######### -# See whether we should enable JSON1 -AC_ARG_ENABLE(json1, AC_HELP_STRING([--enable-json1],[Enable the JSON1 extension])) -AC_MSG_CHECKING([whether to support JSON]) -if test "${enable_json1}" = "yes" -o "${enable_all}" = "yes" ; then - OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_JSON1" - AC_MSG_RESULT([yes]) -else - AC_MSG_RESULT([no]) -fi - ######### # See whether we should enable the LIMIT clause on UPDATE and DELETE # statements. diff --git a/doc/json-enhancements.md b/doc/json-enhancements.md new file mode 100644 index 000000000..bc03e8978 --- /dev/null +++ b/doc/json-enhancements.md @@ -0,0 +1,144 @@ +# JSON Functions Enhancements (2022) + +This document summaries enhancements to the SQLite JSON support added in +early 2022. + +## 1.0 Change summary: + + 1. New **->** and **->>** operators that work like MySQL and PostgreSQL (PG). + 2. JSON functions are built-in rather than being an extension. They + are included by default, but can be omitted using the + -DSQLITE_OMIT_JSON compile-time option. + + +## 2.0 New operators **->** and **->>** + +The SQLite language adds two new binary operators **->** and **->>**. +Both operators are similar to json_extract(). The left operand is +JSON and the right operand is a JSON path expression (possibly abbreviated +for compatibility with PG - see below). So they are similar to a +two-argument call to json_extract(). + +The difference between -> and ->> (and json_extract()) is as follows: + + * The -> operator always returns JSON. + + * The ->> operator converts the answer into a primitive SQL datatype + such as TEXT, INTEGER, REAL, or NULL. If a JSON object or array + is selected, that object or array is rendered as text. If a JSON + value is selected, that value is converted into its corresponding + SQL type + + * The json_extract() interface returns JSON when a JSON object or + array is selected, or a primitive SQL datatype when a JSON value + is selected. This is different from MySQL, in which json_extract() + always returns JSON, but the difference is retained because it has + worked that way for 6 years and changing it now would likely break + a lot of legacy code. + +In MySQL and PG, the ->> operator always returns TEXT (or NULL) and never +INTEGER or REAL. This is due to limitations in the type handling capabilities +of those systems. In MySQL and PG, the result type a function or operator +may only depend on the type of its arguments, never the value of its arguments. +But the underlying JSON type depends on the value of the JSON path +expression, not the type of the JSON path expression (which is always TEXT). +Hence, the result type of ->> in MySQL and PG is unable to vary according +to the type of the JSON value being extracted. + +The type system in SQLite is more general. Functions in SQLite are able +to return different datatypes depending on the value of their arguments. +So the ->> operator in SQLite is able to return TEXT, INTEGER, REAL, or NULL +depending on the JSON type of the value being extracted. This means that +the behavior of the ->> is slightly different in SQLite versus MySQL and PG +in that it will sometimes return INTEGER and REAL values, depending on its +inputs. It is possible to implement the ->> operator in SQLite so that it +always operates exactly like MySQL and PG and always returns TEXT or NULL, +but I have been unable to think of any situations where returning the +actual JSON value this would cause problems, so I'm including the enhanced +functionality in SQLite. + +The table below attempts to summarize the differences between the +-> and ->> operators and the json_extract() function, for SQLite, MySQL, +and PG. JSON values are shown using their SQL text representation but +in a bold font. + + + +
JSONPATH-> operator
(all)		->> operator
(MySQL/PG) + 		->> operator
(SQLite)		json_extract()
(SQLite) +
**'{"a":123}'** '$.a' **'123'** '123' 123 123 +
**'{"a":4.5}'** '$.a' **'4.5'** '4.5' 4.5 4.5 +
**'{"a":"xyz"}'** '$.a' **'"xyz"'** 'xyz' 'xyz' 'xyz' +
**'{"a":null}'** '$.a' **'null'** NULL NULL NULL +
**'{"a":[6,7,8]}'** '$.a' **'[6,7,8]'** '[6,7,8]' '[6,7,8]' **'[6,7,8]'** +
**'{"a":{"x":9}}'** '$.a' **'{"x":9}'** '{"x":9}' '{"x":9}' **'{"x":9}'** +
**'{"b":999}'** '$.a' NULL NULL NULL NULL +
+ +Important points about the table above: + + * The -> operator always returns either JSON or NULL. + + * The ->> operator never returns JSON. It always returns TEXT or NULL, or in the + case of SQLite, INTEGER or REAL. + + * The MySQL json_extract() function works exactly the same + as the MySQL -> operator. + + * The SQLite json_extract() operator works like -> for JSON objects and + arrays, and like ->> for JSON values. + + * The -> operator works the same for all systems. + + * The only difference in ->> between SQLite and other systems is that + when the JSON value is numeric, SQLite returns a numeric SQL value, + whereas the other systems return a text representation of the numeric + value. + +### 2.1 Abbreviated JSON path expressions for PG compatibility + +The table above always shows the full JSON path expression: '$.a'. But +PG does not accept this syntax. PG only allows a single JSON object label +name or a single integer array index. In order to provide compatibility +with PG, The -> and ->> operators in SQLite are extended to also support +a JSON object label or an integer array index for the right-hand side +operand, in addition to a full JSON path expression. + +Thus, a -> or ->> operator that works on MySQL will work in +SQLite. And a -> or ->> operator that works in PG will work in SQLite. +But because SQLite supports the union of the disjoint capabilities of +MySQL and PG, there will always be -> and ->> operators that work in +SQLite that do not work in one of MySQL and PG. This is an unavoidable +consequence of the different syntax for -> and ->> in MySQL and PG. + +In the following table, assume that "value1" is a JSON object and +"value2" is a JSON array. + + +
SQL expression Works in MySQL?Works in PG?Works in SQLite +
value1->'$.a' yes no yes +
value1->'a' no yes yes +
value2->'$[2]' yes no yes +
value2->2 no yes yes +
+ +The abbreviated JSON path expressions only work for the -> and ->> operators +in SQLite. The json_extract() function, and all other built-in SQLite +JSON functions, continue to require complete JSON path expressions for their +PATH arguments. + +## 3.0 JSON moved into the core + +The JSON interface is now moved into the SQLite core. + +When originally written in 2015, the JSON functions were an extension +that could be optionally included at compile-time, or loaded at run-time. +The implementation was in a source file named ext/misc/json1.c in the +source tree. JSON functions were only compiled in if the +-DSQLITE_ENABLE_JSON1 compile-time option was used. + +After these enhancements, the JSON functions are now built-ins. +The source file that implements the JSON functions is moved to src/json.c. +No special compile-time options are needed to load JSON into the build. +Instead, there is a new -DSQLITE_OMIT_JSON compile-time option to leave +them out. diff --git a/ext/expert/sqlite3expert.c b/ext/expert/sqlite3expert.c index a5eb109b4..f29b8a9e0 100644 --- a/ext/expert/sqlite3expert.c +++ b/ext/expert/sqlite3expert.c @@ -697,17 +697,25 @@ static int idxGetTableInfo( ){ sqlite3_stmt *p1 = 0; int nCol = 0; - int nTab = STRLEN(zTab); - int nByte = sizeof(IdxTable) + nTab + 1; + int nTab; + int nByte; IdxTable *pNew = 0; int rc, rc2; char *pCsr = 0; int nPk = 0; + *ppOut = 0; + if( zTab==0 ) return SQLITE_ERROR; + nTab = STRLEN(zTab); + nByte = sizeof(IdxTable) + nTab + 1; rc = idxPrintfPrepareStmt(db, &p1, pzErrmsg, "PRAGMA table_xinfo=%Q", zTab); while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(p1) ){ const char *zCol = (const char*)sqlite3_column_text(p1, 1); const char *zColSeq = 0; + if( zCol==0 ){ + rc = SQLITE_ERROR; + break; + } nByte += 1 + STRLEN(zCol); rc = sqlite3_table_column_metadata( db, "main", zTab, zCol, 0, &zColSeq, 0, 0, 0 @@ -734,7 +742,9 @@ static int idxGetTableInfo( while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(p1) ){ const char *zCol = (const char*)sqlite3_column_text(p1, 1); const char *zColSeq = 0; - int nCopy = STRLEN(zCol) + 1; + int nCopy; + if( zCol==0 ) continue; + nCopy = STRLEN(zCol) + 1; pNew->aCol[nCol].zName = pCsr; pNew->aCol[nCol].iPk = (sqlite3_column_int(p1, 5)==1 && nPk==1); memcpy(pCsr, zCol, nCopy); @@ -886,6 +896,7 @@ static int idxFindCompatible( IdxConstraint *pT = pTail; sqlite3_stmt *pInfo = 0; const char *zIdx = (const char*)sqlite3_column_text(pIdxList, 1); + if( zIdx==0 ) continue; /* Zero the IdxConstraint.bFlag values in the pEq list */ for(pIter=pEq; pIter; pIter=pIter->pLink) pIter->bFlag = 0; @@ -1168,7 +1179,7 @@ static void idxWriteFree(IdxWrite *pTab){ ** runs all the queries to see which indexes they prefer, and populates ** IdxStatement.zIdx and IdxStatement.zEQP with the results. */ -int idxFindIndexes( +static int idxFindIndexes( sqlite3expert *p, char **pzErr /* OUT: Error message (sqlite3_malloc) */ ){ @@ -1297,6 +1308,7 @@ static int idxProcessOneTrigger( rc = idxPrintfPrepareStmt(p->db, &pSelect, pzErr, zSql, zTab, zTab); while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSelect) ){ const char *zCreate = (const char*)sqlite3_column_text(pSelect, 0); + if( zCreate==0 ) continue; rc = sqlite3_exec(p->dbv, zCreate, 0, 0, pzErr); } idxFinalize(&rc, pSelect); @@ -1399,8 +1411,9 @@ static int idxCreateVtabSchema(sqlite3expert *p, char **pzErrmsg){ const char *zName = (const char*)sqlite3_column_text(pSchema, 1); const char *zSql = (const char*)sqlite3_column_text(pSchema, 2); + if( zType==0 || zName==0 ) continue; if( zType[0]=='v' || zType[1]=='r' ){ - rc = sqlite3_exec(p->dbv, zSql, 0, 0, pzErrmsg); + if( zSql ) rc = sqlite3_exec(p->dbv, zSql, 0, 0, pzErrmsg); }else{ IdxTable *pTab; rc = idxGetTableInfo(p->db, zName, &pTab, pzErrmsg); @@ -1537,6 +1550,7 @@ static void idxRemFunc( case SQLITE_BLOB: case SQLITE_TEXT: { int nByte = sqlite3_value_bytes(argv[1]); + const void *pData = 0; if( nByte>pSlot->nByte ){ char *zNew = (char*)sqlite3_realloc(pSlot->z, nByte*2); if( zNew==0 ){ @@ -1548,9 +1562,11 @@ static void idxRemFunc( } pSlot->n = nByte; if( pSlot->eType==SQLITE_BLOB ){ - memcpy(pSlot->z, sqlite3_value_blob(argv[1]), nByte); + pData = sqlite3_value_blob(argv[1]); + if( pData ) memcpy(pSlot->z, pData, nByte); }else{ - memcpy(pSlot->z, sqlite3_value_text(argv[1]), nByte); + pData = sqlite3_value_text(argv[1]); + memcpy(pSlot->z, pData, nByte); } break; } @@ -1761,6 +1777,7 @@ static int idxPopulateStat1(sqlite3expert *p, char **pzErr){ i64 iRowid = sqlite3_column_int64(pAllIndex, 0); const char *zTab = (const char*)sqlite3_column_text(pAllIndex, 1); const char *zIdx = (const char*)sqlite3_column_text(pAllIndex, 2); + if( zTab==0 || zIdx==0 ) continue; if( p->iSample<100 && iPrev!=iRowid ){ samplectx.target = (double)p->iSample / 100.0; samplectx.iTarget = p->iSample; @@ -1827,14 +1844,14 @@ sqlite3expert *sqlite3_expert_new(sqlite3 *db, char **pzErrmsg){ /* Copy the entire schema of database [db] into [dbm]. */ if( rc==SQLITE_OK ){ - sqlite3_stmt *pSql; + sqlite3_stmt *pSql = 0; rc = idxPrintfPrepareStmt(pNew->db, &pSql, pzErrmsg, "SELECT sql FROM sqlite_schema WHERE name NOT LIKE 'sqlite_%%'" " AND sql NOT LIKE 'CREATE VIRTUAL %%'" ); while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSql) ){ const char *zSql = (const char*)sqlite3_column_text(pSql, 0); - rc = sqlite3_exec(pNew->dbm, zSql, 0, 0, pzErrmsg); + if( zSql ) rc = sqlite3_exec(pNew->dbm, zSql, 0, 0, pzErrmsg); } idxFinalize(&rc, pSql); } diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 074123d65..097338f54 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -308,6 +308,12 @@ SQLITE_EXTENSION_INIT1 #endif +typedef struct Fts3HashWrapper Fts3HashWrapper; +struct Fts3HashWrapper { + Fts3Hash hash; /* Hash table */ + int nRef; /* Number of pointers to this object */ +}; + static int fts3EvalNext(Fts3Cursor *pCsr); static int fts3EvalStart(Fts3Cursor *pCsr); static int fts3TermSegReaderCursor( @@ -1172,7 +1178,7 @@ static int fts3InitVtab( sqlite3_vtab **ppVTab, /* Write the resulting vtab structure here */ char **pzErr /* Write any error message here */ ){ - Fts3Hash *pHash = (Fts3Hash *)pAux; + Fts3Hash *pHash = &((Fts3HashWrapper*)pAux)->hash; Fts3Table *p = 0; /* Pointer to allocated vtab */ int rc = SQLITE_OK; /* Return code */ int i; /* Iterator variable */ @@ -4007,9 +4013,12 @@ static const sqlite3_module fts3Module = { ** allocated for the tokenizer hash table. */ static void hashDestroy(void *p){ - Fts3Hash *pHash = (Fts3Hash *)p; - sqlite3Fts3HashClear(pHash); - sqlite3_free(pHash); + Fts3HashWrapper *pHash = (Fts3HashWrapper *)p; + pHash->nRef--; + if( pHash->nRef<=0 ){ + sqlite3Fts3HashClear(&pHash->hash); + sqlite3_free(pHash); + } } /* @@ -4039,7 +4048,7 @@ void sqlite3Fts3IcuTokenizerModule(sqlite3_tokenizer_module const**ppModule); */ int sqlite3Fts3Init(sqlite3 *db){ int rc = SQLITE_OK; - Fts3Hash *pHash = 0; + Fts3HashWrapper *pHash = 0; const sqlite3_tokenizer_module *pSimple = 0; const sqlite3_tokenizer_module *pPorter = 0; #ifndef SQLITE_DISABLE_FTS3_UNICODE @@ -4067,23 +4076,24 @@ int sqlite3Fts3Init(sqlite3 *db){ sqlite3Fts3PorterTokenizerModule(&pPorter); /* Allocate and initialize the hash-table used to store tokenizers. */ - pHash = sqlite3_malloc(sizeof(Fts3Hash)); + pHash = sqlite3_malloc(sizeof(Fts3HashWrapper)); if( !pHash ){ rc = SQLITE_NOMEM; }else{ - sqlite3Fts3HashInit(pHash, FTS3_HASH_STRING, 1); + sqlite3Fts3HashInit(&pHash->hash, FTS3_HASH_STRING, 1); + pHash->nRef = 0; } /* Load the built-in tokenizers into the hash table */ if( rc==SQLITE_OK ){ - if( sqlite3Fts3HashInsert(pHash, "simple", 7, (void *)pSimple) - || sqlite3Fts3HashInsert(pHash, "porter", 7, (void *)pPorter) + if( sqlite3Fts3HashInsert(&pHash->hash, "simple", 7, (void *)pSimple) + || sqlite3Fts3HashInsert(&pHash->hash, "porter", 7, (void *)pPorter) #ifndef SQLITE_DISABLE_FTS3_UNICODE - || sqlite3Fts3HashInsert(pHash, "unicode61", 10, (void *)pUnicode) + || sqlite3Fts3HashInsert(&pHash->hash, "unicode61", 10, (void *)pUnicode) #endif #ifdef SQLITE_ENABLE_ICU - || (pIcu && sqlite3Fts3HashInsert(pHash, "icu", 4, (void *)pIcu)) + || (pIcu && sqlite3Fts3HashInsert(&pHash->hash, "icu", 4, (void *)pIcu)) #endif ){ rc = SQLITE_NOMEM; @@ -4092,7 +4102,7 @@ int sqlite3Fts3Init(sqlite3 *db){ #ifdef SQLITE_TEST if( rc==SQLITE_OK ){ - rc = sqlite3Fts3ExprInitTestInterface(db, pHash); + rc = sqlite3Fts3ExprInitTestInterface(db, &pHash->hash); } #endif @@ -4101,23 +4111,26 @@ int sqlite3Fts3Init(sqlite3 *db){ ** module with sqlite. */ if( SQLITE_OK==rc - && SQLITE_OK==(rc = sqlite3Fts3InitHashTable(db, pHash, "fts3_tokenizer")) + && SQLITE_OK==(rc=sqlite3Fts3InitHashTable(db,&pHash->hash,"fts3_tokenizer")) && SQLITE_OK==(rc = sqlite3_overload_function(db, "snippet", -1)) && SQLITE_OK==(rc = sqlite3_overload_function(db, "offsets", 1)) && SQLITE_OK==(rc = sqlite3_overload_function(db, "matchinfo", 1)) && SQLITE_OK==(rc = sqlite3_overload_function(db, "matchinfo", 2)) && SQLITE_OK==(rc = sqlite3_overload_function(db, "optimize", 1)) ){ + pHash->nRef++; rc = sqlite3_create_module_v2( db, "fts3", &fts3Module, (void *)pHash, hashDestroy ); if( rc==SQLITE_OK ){ + pHash->nRef++; rc = sqlite3_create_module_v2( - db, "fts4", &fts3Module, (void *)pHash, 0 + db, "fts4", &fts3Module, (void *)pHash, hashDestroy ); } if( rc==SQLITE_OK ){ - rc = sqlite3Fts3InitTok(db, (void *)pHash); + pHash->nRef++; + rc = sqlite3Fts3InitTok(db, (void *)pHash, hashDestroy); } return rc; } @@ -4126,7 +4139,7 @@ int sqlite3Fts3Init(sqlite3 *db){ /* An error has occurred. Delete the hash table and return the error code. */ assert( rc!=SQLITE_OK ); if( pHash ){ - sqlite3Fts3HashClear(pHash); + sqlite3Fts3HashClear(&pHash->hash); sqlite3_free(pHash); } return rc; diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index 3a62ccc7a..0626486ed 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -641,7 +641,7 @@ int sqlite3Fts3MsrOvfl(Fts3Cursor *, Fts3MultiSegReader *, int *); int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr); /* fts3_tokenize_vtab.c */ -int sqlite3Fts3InitTok(sqlite3*, Fts3Hash *); +int sqlite3Fts3InitTok(sqlite3*, Fts3Hash *, void(*xDestroy)(void*)); /* fts3_unicode2.c (functions generated by parsing unicode text files) */ #ifndef SQLITE_DISABLE_FTS3_UNICODE diff --git a/ext/fts3/fts3_tokenize_vtab.c b/ext/fts3/fts3_tokenize_vtab.c index 8bd22230c..65d7eef4c 100644 --- a/ext/fts3/fts3_tokenize_vtab.c +++ b/ext/fts3/fts3_tokenize_vtab.c @@ -420,7 +420,7 @@ static int fts3tokRowidMethod( ** Register the fts3tok module with database connection db. Return SQLITE_OK ** if successful or an error code if sqlite3_create_module() fails. */ -int sqlite3Fts3InitTok(sqlite3 *db, Fts3Hash *pHash){ +int sqlite3Fts3InitTok(sqlite3 *db, Fts3Hash *pHash, void(*xDestroy)(void*)){ static const sqlite3_module fts3tok_module = { 0, /* iVersion */ fts3tokConnectMethod, /* xCreate */ @@ -449,7 +449,9 @@ int sqlite3Fts3InitTok(sqlite3 *db, Fts3Hash *pHash){ }; int rc; /* Return code */ - rc = sqlite3_create_module(db, "fts3tokenize", &fts3tok_module, (void*)pHash); + rc = sqlite3_create_module_v2( + db, "fts3tokenize", &fts3tok_module, (void*)pHash, xDestroy + ); return rc; } diff --git a/ext/fts3/mkfts3amal.tcl b/ext/fts3/mkfts3amal.tcl deleted file mode 100644 index 059048717..000000000 --- a/ext/fts3/mkfts3amal.tcl +++ /dev/null @@ -1,115 +0,0 @@ -#!/usr/bin/tclsh -# -# This script builds a single C code file holding all of FTS3 code. -# The name of the output file is fts3amal.c. To build this file, -# first do: -# -# make target_source -# -# The make target above moves all of the source code files into -# a subdirectory named "tsrc". (This script expects to find the files -# there and will not work if they are not found.) -# -# After the "tsrc" directory has been created and populated, run -# this script: -# -# tclsh mkfts3amal.tcl -# -# The amalgamated FTS3 code will be written into fts3amal.c -# - -# Open the output file and write a header comment at the beginning -# of the file. -# -set out [open fts3amal.c w] -set today [clock format [clock seconds] -format "%Y-%m-%d %H:%M:%S UTC" -gmt 1] -puts $out [subst \ -{/****************************************************************************** -** This file is an amalgamation of separate C source files from the SQLite -** Full Text Search extension 2 (fts3). By combining all the individual C -** code files into this single large file, the entire code can be compiled -** as a one translation unit. This allows many compilers to do optimizations -** that would not be possible if the files were compiled separately. It also -** makes the code easier to import into other projects. -** -** This amalgamation was generated on $today. -*/}] - -# These are the header files used by FTS3. The first time any of these -# files are seen in a #include statement in the C code, include the complete -# text of the file in-line. The file only needs to be included once. -# -foreach hdr { - fts3.h - fts3_hash.h - fts3_tokenizer.h - sqlite3.h - sqlite3ext.h -} { - set available_hdr($hdr) 1 -} - -# 78 stars used for comment formatting. -set s78 \ -{*****************************************************************************} - -# Insert a comment into the code -# -proc section_comment {text} { - global out s78 - set n [string length $text] - set nstar [expr {60 - $n}] - set stars [string range $s78 0 $nstar] - puts $out "/************** $text $stars/" -} - -# Read the source file named $filename and write it into the -# sqlite3.c output file. If any #include statements are seen, -# process them approprately. -# -proc copy_file {filename} { - global seen_hdr available_hdr out - set tail [file tail $filename] - section_comment "Begin file $tail" - set in [open $filename r] - while {![eof $in]} { - set line [gets $in] - if {[regexp {^#\s*include\s+["<]([^">]+)[">]} $line all hdr]} { - if {[info exists available_hdr($hdr)]} { - if {$available_hdr($hdr)} { - section_comment "Include $hdr in the middle of $tail" - copy_file tsrc/$hdr - section_comment "Continuing where we left off in $tail" - } - } elseif {![info exists seen_hdr($hdr)]} { - set seen_hdr($hdr) 1 - puts $out $line - } - } elseif {[regexp {^#ifdef __cplusplus} $line]} { - puts $out "#if 0" - } elseif {[regexp {^#line} $line]} { - # Skip #line directives. - } else { - puts $out $line - } - } - close $in - section_comment "End of $tail" -} - - -# Process the source files. Process files containing commonly -# used subroutines first in order to help the compiler find -# inlining opportunities. -# -foreach file { - fts3.c - fts3_hash.c - fts3_porter.c - fts3_tokenizer.c - fts3_tokenizer1.c -} { - copy_file tsrc/$file -} - -close $out diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index fe253984b..6b88f31c0 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -5565,7 +5565,7 @@ int sqlite3Fts5IndexQuery( if( sqlite3Fts5BufferSize(&p->rc, &buf, nToken+1)==0 ){ int iIdx = 0; /* Index to search */ int iPrefixIdx = 0; /* +1 prefix index */ - if( nToken ) memcpy(&buf.p[1], pToken, nToken); + if( nToken>0 ) memcpy(&buf.p[1], pToken, nToken); /* Figure out which index to search and set iIdx accordingly. If this ** is a prefix query for which there is no prefix index, set iIdx to diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c index fd3a9066b..9a8b2fadb 100644 --- a/ext/fts5/fts5_main.c +++ b/ext/fts5/fts5_main.c @@ -804,7 +804,7 @@ static int fts5SorterNext(Fts5Cursor *pCsr){ rc = sqlite3_step(pSorter->pStmt); if( rc==SQLITE_DONE ){ rc = SQLITE_OK; - CsrFlagSet(pCsr, FTS5CSR_EOF); + CsrFlagSet(pCsr, FTS5CSR_EOF|FTS5CSR_REQUIRE_CONTENT); }else if( rc==SQLITE_ROW ){ const u8 *a; const u8 *aBlob; diff --git a/ext/fts5/test/fts5corrupt3.test b/ext/fts5/test/fts5corrupt3.test index adfaa6d85..f9a95665c 100644 --- a/ext/fts5/test/fts5corrupt3.test +++ b/ext/fts5/test/fts5corrupt3.test @@ -15366,6 +15366,160 @@ do_catchsql_test 79.2 { INSERT INTO t1(t1) SELECT 'merge' FROM t2; } {1 {query aborted}} +#------------------------------------------------------------------------- +reset_db +do_test 80.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +.open --hexdb +| size 40960 pagesize 4096 filename crash-f928a9c1ec68dd.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 0a .....@ ........ +| 32: 00 00 00 00 00 00 00 00 00 00 00 0d 00 00 00 04 ................ +| 96: 00 00 00 00 0d 00 00 00 0d 0b 6e 00 0f a3 0f 4c ..........n....L +| 112: 0e e1 0e 81 0e 24 0d cc 0d 72 0d 1b 0c b0 0c 50 .....$...r.....P +| 128: 0b f8 0b b3 0b 6e 00 00 00 00 00 00 00 00 00 00 .....n.......... +| 2912: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 43 0d ..............C. +| 2928: 06 17 11 11 08 75 74 61 62 6c 65 74 34 74 34 43 .....utablet4t4C +| 2944: 52 45 41 54 45 20 56 49 52 54 55 41 4c 20 54 41 REATE VIRTUAL TA +| 2960: 42 4c 45 20 74 34 20 55 53 49 4e 47 20 66 74 73 BLE t4 USING fts +| 2976: 35 76 6f 63 61 62 28 27 74 32 27 2c 20 27 72 6f 5vocab('t2', 'ro +| 2992: 77 27 29 43 0c 06 17 11 11 08 75 74 61 62 6c 65 w')C......utable +| 3008: 74 33 74 33 43 52 45 41 54 45 20 56 49 52 54 55 t3t3CREATE VIRTU +| 3024: 41 4c 20 54 41 42 4c 45 20 74 33 20 55 53 49 4e AL TABLE t3 USIN +| 3040: 47 20 66 74 73 35 76 6f 63 61 62 28 27 74 31 27 G fts5vocab('t1' +| 3056: 2c 20 27 72 6f 77 27 29 56 0b 06 17 1f 1f 01 7d , 'row')V....... +| 3072: 74 61 62 6c 65 74 32 5f 63 6f 6e 66 69 67 74 32 tablet2_configt2 +| 3088: 5f 63 6f 6e 66 69 67 0a 43 52 45 41 54 45 20 54 _config.CREATE T +| 3104: 41 42 4c 45 20 27 74 32 5f 63 6f 6e 66 69 67 27 ABLE 't2_config' +| 3120: 28 6b 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 (k PRIMARY KEY, +| 3136: 76 29 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 v) WITHOUT ROWID +| 3152: 5e 0a 07 17 21 21 01 81 07 74 61 62 6c 65 74 32 ^...!!...tablet2 +| 3168: 5f 63 6f 6e 74 65 6e 74 74 32 5f 63 6f 6e 74 65 _contentt2_conte +| 3184: 6e 74 09 43 52 45 41 54 45 20 54 41 42 4c 45 20 nt.CREATE TABLE +| 3200: 27 74 32 5f 63 6f 6e 74 65 6e 74 27 28 69 64 20 't2_content'(id +| 3216: 49 4e 54 45 47 45 52 20 50 52 49 4d 41 52 59 20 INTEGER PRIMARY +| 3232: 4b 45 59 2c 20 63 30 2c 20 63 31 2c 20 63 32 29 KEY, c0, c1, c2) +| 3248: 69 09 07 17 19 19 01 81 2d 74 61 62 6c 65 74 32 i.......-tablet2 +| 3264: 5f 69 64 78 74 32 5f 69 64 78 08 43 52 45 41 54 _idxt2_idx.CREAT +| 3280: 45 20 54 41 42 4c 45 20 27 74 32 5f 69 64 78 27 E TABLE 't2_idx' +| 3296: 28 73 65 67 69 64 2c 20 74 65 72 6d 2c 20 70 67 (segid, term, pg +| 3312: 6e 6f 2c 20 50 52 49 4d 41 52 59 20 4b 45 59 28 no, PRIMARY KEY( +| 3328: 73 65 67 69 64 2c 20 74 65 72 6d 29 29 20 57 49 segid, term)) WI +| 3344: 54 48 4f 55 54 20 52 4f 57 49 44 55 08 07 17 1b THOUT ROWIDU.... +| 3360: 1b 01 81 01 74 61 62 6c 65 74 32 5f 64 61 74 61 ....tablet2_data +| 3376: 74 32 5f 64 61 74 61 07 43 52 45 41 54 45 20 54 t2_data.CREATE T +| 3392: 41 42 4c 45 20 27 74 32 5f 64 61 74 61 27 28 69 ABLE 't2_data'(i +| 3408: 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d 41 52 d INTEGER PRIMAR +| 3424: 59 20 4b 45 59 2c 20 62 6c 6f 63 6b 20 42 4c 4f Y KEY, block BLO +| 3440: 42 29 58 07 07 17 11 11 08 81 1d 74 61 62 6c 65 B)X........table +| 3456: 74 32 74 32 43 52 45 41 54 45 20 56 49 52 54 55 t2t2CREATE VIRTU +| 3472: 41 4c 20 54 41 42 4c 45 20 74 32 20 55 53 49 4e AL TABLE t2 USIN +| 3488: 47 20 66 74 73 35 28 27 61 27 2c 5b 62 5d 2c 22 G fts5('a',[b],. +| 3504: 63 22 2c 64 65 74 61 69 6c 3d 6e 6f 6e 65 2c 63 c.,detail=none,c +| 3520: 6f 6c 75 6d 6e 73 69 7a 65 3d 30 29 56 06 06 17 olumnsize=0)V... +| 3536: 1f 1f 01 7d 74 61 62 6c 65 74 31 5f 63 6f 6e 66 ....tablet1_conf +| 3552: 69 67 74 31 5f 63 6f 6e 66 69 67 06 43 52 45 41 igt1_config.CREA +| 3568: 54 45 20 54 41 42 4c 45 20 27 74 31 5f 63 6f 6e TE TABLE 't1_con +| 3584: 66 69 67 27 28 6b 20 50 52 49 4d 41 52 59 20 4b fig'(k PRIMARY K +| 3600: 45 59 2c 20 76 29 20 57 49 54 48 4f 55 54 20 52 EY, v) WITHOUT R +| 3616: 4f 57 49 44 5b 05 07 17 21 21 01 81 01 74 61 62 OWID[...!!...tab +| 3632: 6c 65 74 31 5f 64 6f 63 73 69 7a 65 74 31 5f 64 let1_docsizet1_d +| 3648: 6f 63 73 69 7a 65 05 43 52 45 41 54 45 20 54 41 ocsize.CREATE TA +| 3664: 42 4c 45 20 27 74 31 5f 64 6f 63 73 69 7a 65 27 BLE 't1_docsize' +| 3680: 28 69 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d (id INTEGER PRIM +| 3696: 41 52 59 20 4b 45 59 2c 20 73 7a 20 42 4c 4f 42 ARY KEY, sz BLOB +| 3712: 29 5e 04 07 17 21 21 01 81 07 74 61 62 6c 65 74 )^...!!...tablet +| 3728: 31 5f 63 6f 6e 74 65 6e 74 74 31 5f 63 6f 6e 74 1_contentt1_cont +| 3744: 65 6e 74 04 43 52 45 41 54 45 20 54 41 42 4c 45 ent.CREATE TABLE +| 3760: 20 27 74 31 5f 63 6f 6e 74 65 6e 74 27 28 69 64 't1_content'(id +| 3776: 20 49 4e 54 45 47 45 52 20 50 52 49 4d 41 52 59 INTEGER PRIMARY +| 3792: 20 4b 45 59 2c 20 63 30 2c 20 63 31 2c 20 63 32 KEY, c0, c1, c2 +| 3808: 29 69 03 07 17 19 19 01 81 2d 74 61 62 6c 65 74 )i.......-tablet +| 3824: 31 5f 69 64 78 74 31 5f 69 64 78 03 43 52 45 41 1_idxt1_idx.CREA +| 3840: 54 45 20 54 41 42 4c 45 20 27 74 31 5f 69 64 78 TE TABLE 't1_idx +| 3856: 27 28 73 65 67 69 64 2c 20 74 65 72 6d 2c 20 70 '(segid, term, p +| 3872: 67 6e 6f 2c 20 50 52 49 4d 41 52 59 20 4b 45 59 gno, PRIMARY KEY +| 3888: 28 73 65 67 69 64 2c 20 74 65 72 6d 29 29 20 57 (segid, term)) W +| 3904: 49 54 48 4f 55 54 20 52 4f 57 49 44 55 02 07 17 ITHOUT ROWIDU... +| 3920: 1b 1b 01 81 01 74 61 62 6c 65 74 31 5f 64 61 74 .....tablet1_dat +| 3936: 61 74 31 5f 64 61 74 61 02 43 52 45 41 54 45 20 at1_data.CREATE +| 3952: 54 41 42 4c 45 20 27 74 31 5f 64 61 74 61 27 28 TABLE 't1_data'( +| 3968: 69 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d 41 id INTEGER PRIMA +| 3984: 52 59 20 4b 45 59 2c 20 62 6c 6f 63 6b 20 42 4c RY KEY, block BL +| 4000: 4f 42 29 5b 01 07 17 11 11 08 81 23 74 61 62 6c OB)[.......#tabl +| 4016: 65 74 31 74 31 43 52 45 41 54 45 20 56 49 52 54 et1t1CREATE VIRT +| 4032: 55 41 4c 20 54 41 42 4c 45 20 74 31 20 55 53 49 UAL TABLE t1 USI +| 4048: 4e 47 20 66 74 73 35 28 61 2c 62 20 75 6e 69 6e NG fts5(a,b unin +| 4064: 64 65 78 65 64 2c 63 2c 74 6f 6b 65 6e 69 7a 65 dexed,c,tokenize +| 4080: 3d 22 70 6f 72 74 65 72 20 61 73 63 69 69 22 29 =.porter ascii.) +| page 2 offset 4096 +| 0: 0d 0f 68 00 05 0f 13 00 0f e6 0f 13 0f a8 0f 7c ..h............| +| 16: 0f 2a 00 00 00 00 00 00 00 00 00 00 00 00 00 00 .*.............. +| 3856: 00 00 00 15 0a 03 00 30 00 00 00 00 01 03 03 00 .......0........ +| 3872: 03 01 01 01 02 01 01 03 01 01 37 8c 80 80 80 80 ..........7..... +| 3888: 01 03 00 74 00 00 00 2e 02 30 61 03 02 02 01 01 ...t.....0a..... +| 3904: 62 03 02 03 01 01 63 03 02 04 01 01 67 03 06 01 b.....c.....g... +| 3920: 02 02 01 01 68 03 06 01 02 03 01 01 69 03 06 01 ....h.......i... +| 3936: 02 04 04 06 06 06 08 08 0f ef 00 14 2a 00 00 00 ............*... +| 3952: 00 01 02 02 00 02 01 01 01 02 01 01 25 88 80 80 ............%... +| 3968: 80 80 01 03 00 50 00 00 00 1f 02 30 67 02 08 02 .....P.....0g... +| 3984: 01 02 02 01 01 68 02 08 03 01 02 03 01 01 69 02 .....h........i. +| 4000: 08 04 01 02 04 04 09 09 37 84 80 80 80 80 01 03 ........7....... +| 4016: 00 74 00 00 00 2e 02 30 61 01 02 02 01 01 62 01 .t.....0a.....b. +| 4032: 02 03 01 01 63 01 02 04 01 01 67 01 06 01 02 02 ....c.....g..... +| 4048: 01 01 68 01 06 01 02 03 01 01 69 01 06 01 02 04 ..h.......i..... +| 4064: 04 06 06 06 08 08 07 01 03 00 14 03 09 00 09 00 ................ +| 4080: 00 00 11 24 00 00 00 00 01 01 01 00 01 01 01 01 ...$............ +| page 3 offset 8192 +| 0: 0a 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 ................ +| 4064: 00 00 00 00 00 00 00 00 00 00 00 00 06 04 01 0c ................ +| 4080: 01 03 02 06 04 01 0c 01 02 02 05 04 09 0c 01 02 ................ +| page 4 offset 12288 +| 0: 0d 00 00 00 03 0f be 00 0f ea 0f d4 0f be 00 00 ................ +| 4016: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 14 03 ................ +| 4032: 05 00 17 17 17 61 20 62 20 63 67 20 68 20 69 67 .....a b cg h ig +| 4048: 20 68 20 69 14 02 05 00 17 17 17 67 20 68 20 69 h i.......g h i +| 4064: 61 20 62 20 63 67 20 68 20 69 14 01 05 00 17 17 a b cg h i...... +| 4080: 17 61 20 62 20 63 64 20 65 20 66 67 20 68 20 69 .a b cd e fg h i +| page 5 offset 16384 +| 0: 0d 00 00 00 03 0f e8 00 0f f8 0f f0 0f e8 00 00 ................ +| 4064: 00 00 00 00 00 00 00 00 06 03 03 00 12 03 00 03 ................ +| 4080: 06 02 03 00 12 03 00 03 06 01 03 00 12 03 00 03 ................ +| page 6 offset 20480 +| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 01 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| page 7 offset 24576 +| 0: 0d 00 00 00 03 0f 9e 00 0f e6 0f ef 00 00 00 00 ................ +| 3984: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 84 ..............A. +| 4000: 80 80 80 80 01 04 00 81 06 00 00 00 34 02 30 61 ............4.0a +| 4016: 01 01 01 01 01 62 01 01 01 01 01 63 01 01 01 01 .....b.....c.... +| 4032: 01 64 01 01 01 65 01 01 01 66 01 01 01 67 01 01 .d...e...f...g.. +| 4048: 01 01 01 68 01 01 01 01 01 69 01 01 01 04 06 06 ...h.....i...... +| 4064: 06 04 04 04 06 06 07 01 03 00 14 03 09 09 09 0f ................ +| 4080: 0a 03 00 24 00 00 00 00 01 01 01 00 01 01 01 01 ...$............ +| page 8 offset 28672 +| 0: 0a 00 00 00 01 0f fa 00 00 00 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 00 00 00 00 00 00 05 04 09 0c 01 12 ................ +| page 9 offset 32768 +| 0: 0d 00 00 00 03 0f be 00 0f ea 0f d4 0f be 00 00 ................ +| 4016: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 14 03 ................ +| 4032: 05 00 17 17 17 61 20 62 20 63 67 20 68 20 69 67 .....a b cg h ig +| 4048: 20 68 20 69 14 02 05 00 17 17 17 67 20 68 20 69 h i.......g h i +| 4064: 61 20 62 20 63 67 20 68 20 69 14 01 05 00 17 17 a b cg h i...... +| 4080: 17 61 20 62 20 63 64 20 65 20 66 67 20 68 20 69 .a b cd e fg h i +| page 10 offset 36864 +| 0: 0a 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 00 00 00 00 ........vers.... +| end crash-f928a9c1ec68dd.db +}]} {} + +do_catchsql_test 80.1 { +SELECT snippet(rowid, -1, '.', '..', '[', '(]'),snippet(rowid, -1, '.', '.', '', '(]'), highlight(t1, 29, 1 , '') FROM t1('g+ h') WHERE rank MATCH 'bm25(1.0, 10)' ORDER BY NOT (SELECT 1 FROM t1('g+ æ') WHERE rank MATCH 'bm25(1.0, 10)' ORDER BY rank); +} {1 {database disk image is malformed}} + + sqlite3_fts5_may_be_corrupt 0 finish_test diff --git a/ext/fts5/test/fts5eb.test b/ext/fts5/test/fts5eb.test index 9d6f251ed..ce40e471a 100644 --- a/ext/fts5/test/fts5eb.test +++ b/ext/fts5/test/fts5eb.test @@ -83,7 +83,7 @@ for {set i 0} {$i < 255} {incr i} { do_execsql_test 3.0 { CREATE VIRTUAL TABLE e1 USING fts5(text, tokenize = 'porter unicode61'); - INSERT INTO e1 VALUES ("just a few words with a / inside"); + INSERT INTO e1 VALUES ('just a few words with a / inside'); } do_execsql_test 3.1 { SELECT rowid, bm25(e1) FROM e1 WHERE e1 MATCH '"just"' ORDER BY rank; diff --git a/ext/fts5/test/fts5integrity.test b/ext/fts5/test/fts5integrity.test index d922ad3b8..403883086 100644 --- a/ext/fts5/test/fts5integrity.test +++ b/ext/fts5/test/fts5integrity.test @@ -188,11 +188,11 @@ foreach {tn pgsz} { INSERT INTO hh(hh, rank) VALUES('pgsz', $pgsz); WITH s(i) AS (SELECT 0 UNION ALL SELECT i+1 FROM s WHERE i<999) - INSERT INTO hh SELECT printf("%.3d%.3d%.3d %.3d%.3d%.3d",i,i,i,i+1,i+1,i+1) + INSERT INTO hh SELECT printf('%.3d%.3d%.3d %.3d%.3d%.3d',i,i,i,i+1,i+1,i+1) FROM s; WITH s(i) AS (SELECT 0 UNION ALL SELECT i+1 FROM s WHERE i<999) - INSERT INTO hh SELECT printf("%.3d%.3d%.3d %.3d%.3d%.3d",i,i,i,i+1,i+1,i+1) + INSERT INTO hh SELECT printf('%.3d%.3d%.3d %.3d%.3d%.3d',i,i,i,i+1,i+1,i+1) FROM s; INSERT INTO hh(hh) VALUES('optimize'); diff --git a/ext/lsm1/lsm_vtab.c b/ext/lsm1/lsm_vtab.c index f96a6bbd7..bb1460297 100644 --- a/ext/lsm1/lsm_vtab.c +++ b/ext/lsm1/lsm_vtab.c @@ -26,7 +26,7 @@ ** ** The virtual table contains read-only hidden columns: ** -** lsm1_key A BLOB which is the raw LSM key. If the "keytype" +** lsm1_key A BLOB which is the raw LSM key. If the "keytype" ** is BLOB or TEXT then this column is exactly the ** same as the key. For the UINT keytype, this column ** will be a variable-length integer encoding of the key. diff --git a/ext/misc/csv.c b/ext/misc/csv.c index d08ddd6b3..b51fbad30 100644 --- a/ext/misc/csv.c +++ b/ext/misc/csv.c @@ -941,7 +941,7 @@ int sqlite3_csv_init( char **pzErrMsg, const sqlite3_api_routines *pApi ){ -#ifndef SQLITE_OMIT_VIRTUALTABLE +#ifndef SQLITE_OMIT_VIRTUALTABLE int rc; SQLITE_EXTENSION_INIT2(pApi); rc = sqlite3_create_module(db, "csv", &CsvModule, 0); diff --git a/ext/misc/fileio.c b/ext/misc/fileio.c index c9988f607..7cdbd5968 100644 --- a/ext/misc/fileio.c +++ b/ext/misc/fileio.c @@ -368,10 +368,11 @@ static int writeFile( mode_t mode, /* MODE parameter passed to writefile() */ sqlite3_int64 mtime /* MTIME parameter (or -1 to not set time) */ ){ + if( zFile==0 ) return 1; #if !defined(_WIN32) && !defined(WIN32) if( S_ISLNK(mode) ){ const char *zTo = (const char*)sqlite3_value_text(pData); - if( symlink(zTo, zFile)<0 ) return 1; + if( zTo==0 || symlink(zTo, zFile)<0 ) return 1; }else #endif { diff --git a/ext/misc/ieee754.c b/ext/misc/ieee754.c index 66d946f3d..ff5d2d333 100644 --- a/ext/misc/ieee754.c +++ b/ext/misc/ieee754.c @@ -284,7 +284,7 @@ int sqlite3_ieee_init( SQLITE_EXTENSION_INIT2(pApi); (void)pzErrMsg; /* Unused parameter */ for(i=0; i